We Keep Coding

How to avoid skipping blank rows or columns in Apache POI

When I parse the file using Apace POI, the empty rows are getting skipped and a List of String arrays with non-empty rows are being returned.
How can I tell the API, not to skip reading the empty row or columns ?
The code to read is somehow skipping the rows with no data.
XExcelFileReader fileReader = new XExcelFileReader(excelByteArray, sheetFromWhichToRead);
dataFromFile = fileReader.readRows();
is used to read the data from the class XExcelFileReader. The variable dataFromFile is a List of String array.
This is the code to read the data row:
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Pattern;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.model.StylesTable;
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
/**
* This class is responsible for reading the row contents for a given byte array and sheet name
*/
public class XExcelFileReader {
/**
* This is the Row Num of the current row that was read
*/
private int rowNum = 0;
/**
* The OPCPackage is the package used to laod the Input Stream used for only .xlsx Files
*/
private OPCPackage opcPkg;
/**
* These are the String Tables that are read from the Excel File
*/
private ReadOnlySharedStringsTable stringsTable;
/**
* The XML Streaming API will be used
*/
private XMLStreamReader xmlReader;
/** The styles table which has formatting information about cells. */
private StylesTable styles;
/**
* #param excelByteArray the excel byte array
* #param sheetFromWhichToRead the excel sheet from which to read
* #throws Exception the exception
*/
public XExcelFileReader(final byte[] excelByteArray, final String sheetFromWhichToRead) throws Exception {
InputStream excelStream = new ByteArrayInputStream(excelByteArray);
opcPkg = OPCPackage.open(excelStream);
this.stringsTable = new ReadOnlySharedStringsTable(opcPkg);
XSSFReader xssfReader = new XSSFReader(opcPkg);
styles = xssfReader.getStylesTable();
XMLInputFactory factory = XMLInputFactory.newInstance();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData();
InputStream inputStream = null;
while (iter.hasNext()) {
inputStream = iter.next();
String tempSheetName = iter.getSheetName();
if (StringUtils.isNotEmpty(tempSheetName)) {
tempSheetName = tempSheetName.trim();
if (tempSheetName.equals(sheetFromWhichToRead)) {
break;
}
}
}
xmlReader = factory.createXMLStreamReader(inputStream);
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals("sheetData")) {
break;
}
}
}
}
/**
* #return rowNum
*/
public final int rowNum() {
return rowNum;
}
/**
* #return List<String[]> List of String array which can hold the content
* #throws XMLStreamException the XMLStreamException
*/
public final List<String[]> readRows() throws XMLStreamException {
String elementName = "row";
List<String[]> dataRows = new ArrayList<String[]>();
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals(elementName)) {
rowNum++;
dataRows.add(getDataRow());
// TODO need to see if the batch Size is required
// if (dataRows.size() == batchSize)
// break;
}
}
}
return dataRows;
}
/**
* #return String [] of Row Data
* #throws XMLStreamException the XMLStreamException
*/
private String[] getDataRow() throws XMLStreamException {
List<String> rowValues = new ArrayList<String>();
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals("c")) {
CellReference cellReference = new CellReference(xmlReader.getAttributeValue(null, "r"));
// Fill in the possible blank cells!
while (rowValues.size() < cellReference.getCol()) {
rowValues.add("");
}
String cellType = xmlReader.getAttributeValue(null, "t");
String cellStyleStr = xmlReader.getAttributeValue(null, "s");
rowValues.add(getCellValue(cellType, cellStyleStr));
}
} else if (xmlReader.isEndElement() && xmlReader.getLocalName().equals("row")) {
break;
}
}
return rowValues.toArray(new String[rowValues.size()]);
}
/**
* #param cellType the cell type
* #param cellStyleStr the cell style
* #return cell content the cell value
* #throws XMLStreamException the XMLStreamException
*/
private String getCellValue(final String cellType, final String cellStyleStr) throws XMLStreamException {
String value = ""; // by default
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals("v")) {
if (cellType != null && cellType.equals("s")) {
int idx = Integer.parseInt(xmlReader.getElementText());
String s = stringsTable.getEntryAt(idx);
return new XSSFRichTextString(s).toString();
}
if (cellStyleStr != null) {
String cellValue = xmlReader.getElementText();
int styleIndex = Integer.parseInt(cellStyleStr);
XSSFCellStyle style = styles.getStyleAt(styleIndex);
short formatIndex = style.getDataFormat();
if (!isValidDouble(cellValue)) {
return cellValue;
}
double doubleVal = Double.valueOf(cellValue);
boolean isValidExcelDate = HSSFDateUtil.isInternalDateFormat(formatIndex);
Date date = null;
if (isValidExcelDate) {
date = HSSFDateUtil.getJavaDate(doubleVal);
String dateStr = dateAsString(date);
return dateStr;
}
if (!(doubleVal == Math.floor(doubleVal))) {
return Double.toString(doubleVal);
}
return cellValue;
}
else {
return xmlReader.getElementText();
}
}
} else if (xmlReader.isEndElement() && xmlReader.getLocalName().equals("c")) {
break;
}
}
return value;
}
/**
* To check whether the incoming value can be used in the Double utility method Double.valueOf() to prevent
* NumberFormatException.
* #param stringVal - String to be validated.
* #return - true if it is a valid String which can be passed into Double.valueOf method. <br/> For more information
* refer- <a>https://docs.oracle.com/javase/7/docs/api/java/lang/Double.html#valueOf(java.lang.String)</a>
*/
private boolean isValidDouble(String stringVal) {
final String Digits = "(\\p{Digit}+)";
final String HexDigits = "(\\p{XDigit}+)";
// an exponent is 'e' or 'E' followed by an optionally
// signed decimal integer.
final String Exp = "[eE][+-]?" + Digits;
final String fpRegex = ("[\\x00-\\x20]*" + // Optional leading "whitespace"
"[+-]?(" + // Optional sign character
"NaN|" + // "NaN" string
"Infinity|" + // "Infinity" string
// A decimal floating-point string representing a finite positive
// number without a leading sign has at most five basic pieces:
// Digits . Digits ExponentPart FloatTypeSuffix
//
// Since this method allows integer-only strings as input
// in addition to strings of floating-point literals, the
// two sub-patterns below are simplifications of the grammar
// productions from the Java Language Specification, 2nd
// edition, section 3.10.2.
// Digits ._opt Digits_opt ExponentPart_opt FloatTypeSuffix_opt
"(((" + Digits + "(\\.)?(" + Digits + "?)(" + Exp + ")?)|" +
// . Digits ExponentPart_opt FloatTypeSuffix_opt
"(\\.(" + Digits + ")(" + Exp + ")?)|" +
// Hexadecimal strings
"((" +
// 0[xX] HexDigits ._opt BinaryExponent FloatTypeSuffix_opt
"(0[xX]" + HexDigits + "(\\.)?)|" +
// 0[xX] HexDigits_opt . HexDigits BinaryExponent FloatTypeSuffix_opt
"(0[xX]" + HexDigits + "?(\\.)" + HexDigits + ")" +
")[pP][+-]?" + Digits + "))" + "[fFdD]?))" + "[\\x00-\\x20]*");// Optional trailing "whitespace"
if (Pattern.matches(fpRegex, stringVal))
return true;
else {
return false;
}
}
/**
* Date as string.
* #param date the date
* #return the string
*/
public static String dateAsString(final Date date) {
String dateAsString = null;
if (date != null) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
dateAsString = sdf.format(date);
}
return dateAsString;
}
#Override
protected final void finalize() throws Throwable {
if (opcPkg != null) {
opcPkg.close();
}
super.finalize();
}
}
I want the Empty row also to be part of the List of String array. Just that the String array will be empty.
If it is in the Excel like:
First Row
<< No Data >>
Third Row
then the List should be
dataFromFile
0 ->[First Row]
1 ->[]
2 ->[Third Row]

Here is what the Busy Developers Guide on the POI site says about reading sheets with missing rows:
for (int rowNum = rowStart; rowNum < rowEnd; rowNum++) {
Row r = sheet.getRow(rowNum);
if (r == null) {
// This whole row is empty
// Handle it as needed
continue;
}
int lastColumn = Math.max(r.getLastCellNum(), MY_MINIMUM_COLUMN_COUNT);
for (int cn = 0; cn < lastColumn; cn++) {
Cell c = r.getCell(cn, Row.RETURN_BLANK_AS_NULL);
if (c == null) {
// The spreadsheet is empty in this cell
} else {
// Do something useful with the cell's contents
}
}
}
You really can get a lot of your answers there. You shouldn't ever have to parse the XML directly. Even things that are missing from the high level usermodel usually have a way to get at it through the CT classes generated by XMLBeans.

I am trying to write the result of the dql queries into a separate CSV file using dfc coding.

i have attached screen shot of the resultant CSV file
I am trying to write the result of the dql queries into a separate CSV file using dfc coding. but the result of the first record is printing twice in the resultant sheet.
public class ChkGroupExistence {
/**
* #param args
*/
private static Properties queryProp;
//private static Properties configProp;
IDfSession sess = null;
public ChkGroupExistence() {
System.out.println("Loading Properties..");
LoadProps loadProp = LoadProps.getInstance();
queryProp = loadProp.getQueryProp();
//configProp = loadProp.getConfigProp();
List<String> proj_list = new ArrayList<String>();
List<String> grp_list = new ArrayList<String>();
List<String> acl_list = new ArrayList<String>();
HashMap<String, String> projList_Map = new HashMap<String, String>();
IDfCollection projId_coll = null;
IDfCollection grp_coll = null;
//IDfCollection chk_coll = null;
//IDfCollection acl_coll = null;
String grpqry = null;
String chkqry = null;
//String getACLQuery = null;
int j=0;
CreateSession ifcDocsDfSession = new CreateSession();
try {
sess = ifcDocsDfSession.getSession();
DfLogger.info(this, "Session Created ::" + sess.getSessionId(),
null, null);
} catch (DfException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
String qry = queryProp
.getProperty(IdocsConstants.PROJECT_ID);
try {
CSVWriter csvwriter=new CSVWriter(new FileWriter(new File("C:\\WBG\\IFCTools\\cs_log.csv")));
projId_coll = Util.executeQuery(sess, qry, IDfQuery.READ_QUERY);
while (projId_coll.next()) {
proj_list.add(projId_coll.getString("project_id"));
}
System.out.println("List of Project ::"+proj_list.size());
String tempQuery=queryProp.getProperty(IdocsConstants.P_GROUP_EXIST);
//String tempQuery1=queryProp.getProperty(IdocsConstants.P_GROUP_VERIFY);
//String tempQuery2 = queryProp.getProperty(IdocsConstants.P_GETACL);
List<String[]> csvList=new ArrayList<String[]>();
List<String[]> titleList = new ArrayList<String[]>();
String[] projList;
String[] titleString;
titleString = new String[3];
titleString[0]="ProjectId/Institutionnbr";
titleString[1]="GroupName";
titleString[2]="ACL Name";
titleList.add(titleString);
csvwriter.writeAll(titleList);
for(int i = 0; i <proj_list.size();i++ ) {
//grpqry = tempQuery+proj_list.get(i) + "_ed_off_grp'" ;
grpqry = MessageFormat.format(tempQuery,proj_list.get(i));
//chkqry = queryProp.getProperty(IdocsConstants.P_GROUP_VERIFY);
//System.out.println(grpqry);
//getACLQuery = MessageFormat.format(tempQuery2, proj_list.get(i));
//System.out.println(getACLQuery);
//System.out.println("grp_coll query is executing....");
grp_coll = Util.executeQuery(sess, grpqry, IDfQuery.READ_QUERY);
//System.out.println("verification query is executing.....");
//chk_coll = Util.executeQuery(sess, chkqry, IDfQuery.READ_QUERY);
//acl_coll = Util.executeQuery(sess, getACLQuery, IDfQuery.READ_QUERY);
if (grp_coll!=null && grp_coll.next()) {
String grpName = grp_coll.getString("group_name");
grp_list.add(grpName);
System.out.println("Got group for "+proj_list.get(i)+" :: "+grpName);
projList=new String[3];
projList[0]=proj_list.get(i);
projList[1]=grpName;
//System.out.println(grpName);
projList_Map.put(proj_list.get(i),grp_list.get(j));
j++;
System.out.println(projList_Map.size());
if(chkqry == null){
//System.out.println("group names are adding to the list.....");
//grp_list.add(grpName);
String acl_name = queryProp.getProperty(IdocsConstants.P_GETACL);
acl_list.add(acl_name);
projList[2]=acl_name;
}
csvList.add(projList);
csvwriter.writeAll(csvList);
}
}
System.out.println("Project List is loading....");
Set<String> keySet = projList_Map.keySet();
System.out.println(grp_list);
System.out.println(acl_list);
for(String set : keySet) {
System.out.println(set + " : " +projList_Map.get(set));
}
csvwriter.close();
} catch (DfException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (IndexOutOfBoundsException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String args[])
{
ChkGroupExistence chkexist = new ChkGroupExistence();
}
}
CSVWriter.java
public class CSVWriter {
/**
* Default Delimiter will be used if not given.
*/
private static final char DEF_DELIM = ',';
/**
* Default Quote Character will be used if not given.
*/
private static final char DEF_QUOTE_CHAR = '"';
/**
* Default End of Line Character.
*/
private static final char DEFAULT_EOL = '\n';
/**
* Contains the Delimtter.
*/
private char delimiter;
/**
* Contains the Quote character.
*/
private char quotechar;
/**
* String contains the End of the line character.
*/
private char cLineEnd;
/**
* Instance Variable to hold Write Object.
*/
private Writer rawWriter;
/**
* Instance variable to hold PrintWriter object
*/
private PrintWriter pw;
/**
* Constructor to take File Writer as Input.
*
* #param writer
* File Writer
*/
public CSVWriter(Writer writer) {
this(writer, DEF_DELIM);
}
/**
* Constructor to take File Writer and Delimiter as Input.
*
* #param writer
* File Writer
* #param delim
* Delimiter
*/
public CSVWriter(Writer writer, char delim) {
this(writer, delim, DEF_QUOTE_CHAR);
}
/**
* Constructor to take File Writer, Delimiter and Quote Character as Input.
*
* #param writer
* File Writer
* #param delim
* Delimiter
* #param quote
* Quote Character
*/
public CSVWriter(Writer writer, char delim, char quote) {
this(writer, delim, quote, DEFAULT_EOL);
}
/**
* Constructor to take File Writer, Delimiter, Quote Character and End of
* Line feed as Input.
*
* #param writer
* File Writer
* #param delim
* Delimiter
* #param quote
* #param sEOL
*/
public CSVWriter(Writer writer, char delim, char quote, char sEOL) {
rawWriter = writer;
pw = new PrintWriter(writer);
delimiter = delim;
quotechar = quote;
cLineEnd = sEOL;
}
/**
* Method takes List as input and writes values into the CSV file.
*
* #param list
* List of Cell values.
*/
public void writeAll(List list) {
String sRow[];
for (Iterator iter = list.iterator(); iter.hasNext(); writeNext(sRow)) {
sRow = (String[]) iter.next();
}
}
/**
* Method that takes String[] as input and writes each and every cell.
*
* #param sRow
* String[]
*/
private void writeNext(String sRow[]) {
StringBuffer stringbuffer = new StringBuffer();
for (int i = 0; i < sRow.length; i++) {
if (i != 0) {
stringbuffer.append(delimiter);
}
String s = sRow[i];
if (s == null) {
continue;
}
if (quotechar != 0) {
stringbuffer.append(quotechar);
}
for (int j = 0; j < s.length(); j++) {
char c = s.charAt(j);
if (c == quotechar) {
stringbuffer.append(DEF_QUOTE_CHAR).append(c);
continue;
}
if (c == DEF_QUOTE_CHAR) {
stringbuffer.append(DEF_QUOTE_CHAR).append(c);
} else {
stringbuffer.append(c);
}
}
if (quotechar != 0) {
stringbuffer.append(quotechar);
}
}
stringbuffer.append(cLineEnd);
pw.write(stringbuffer.toString());
}
/**
* Method that closed the Print Writer. Only when this method is called, the
* CSV file will be saved.
*
* #throws IOException
*/
public void close() throws IOException {
pw.flush();
pw.close();
rawWriter.close();
}

Probably because you're including a repeating attribute in your query.
Tip: I tend to include the object ID for reference and debugging when I do this.
You can also insert a "DISTINCT" keyword into your query when including these repeating attributes.
The reason they act like this is the nature of the data model: all repeating attributes are stored in the same table - so when one of these attributes contains multiple values a query could return multiple rows when querying another repeating attribute. In this case a SELECT DISTINCT would do the trick.
You can also play around with the DQL hint ENABLE(ROW_BASED) if you are joining repeating attribute with single value attributes.
Happy coding!

Start reading an Excel sheet at a specified row

I am reading an Excel file using POI's XSSF and SAX (Event API). The Excel sheet has thousands of rows so this is the only way that I have found to have good performance. Now I would like to read Excel file from one particularly row, for example row 6 because these Excel files are updated every day and I store the last row that I have already stored so I can read only the new rows.
How can I start from one row?
private void getExcelField(AcquisitionForm acquisitionForm) throws ExcelReadException, IOException{
InputStream stream=null;
OPCPackage p=null;
try{
p = OPCPackage.open(acquisitionForm.getDatatablePath(), PackageAccess.READ);
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(p);
XSSFReader xssfReader = new XSSFReader(p);
StylesTable styles = xssfReader.getStylesTable();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData();
int index = 0;
//Test with one sheet
iter.hasNext();
//(iter.hasNext()) {
stream = iter.next();
String sheetName = iter.getSheetName();
processSheet(styles, strings, stream, acquisitionForm);
stream.close();
//++index;
//}
p.close();
}catch(Exception e){
throw new ExcelReadException("An error occured during excel file reading ", e);
}finally{
if (stream!=null)
stream.close();
if (p!=null)
p.close();
//++index;
// }
}
}
/**
* Parses and shows the content of one sheet
* using the specified styles and shared-strings tables.
* #param styles
* #param strings
* #param sheetInputStream
* #throws ExcelReadException
*/
private void processSheet(StylesTable styles, ReadOnlySharedStringsTable strings, InputStream sheetInputStream, AcquisitionForm acquisitionForm) throws Exception {
InputSource sheetSource = new InputSource(sheetInputStream);
SAXParserFactory saxFactory = SAXParserFactory.newInstance();
SAXParser saxParser = saxFactory.newSAXParser();
XMLReader sheetParser = saxParser.getXMLReader();
//ContentHandler handler = new MyXSSFSheetHandler(styles, strings);
MyXSSFSheetHandler handler = new MyXSSFSheetHandler(styles, strings,databaseAcquisitionServices, acquisitionForm);
sheetParser.setContentHandler(handler);
sheetParser.parse(sheetSource);
}
MyXSSFSheetHandler
public MyXSSFSheetHandler(StylesTable styles, ReadOnlySharedStringsTable strings,DatabaseAcquisitionServices databaseAcquisitionServices, AcquisitionForm acquisitionForm, int sheetIndex) {
this.stylesTable = styles;
this.sharedStringsTable = strings;
this.formatter = new DataFormatter();
this.value = new StringBuffer();
this.nextDataType = XssfDataType.NUMBER;
this.databaseAcquisitionServices=databaseAcquisitionServices;
this.acquisitionForm=acquisitionForm;
this.sheetIndex = sheetIndex;
}
/**
*
*/
public void startElement(String uri, String localName, String name,
Attributes attributes) throws SAXException {
if ("inlineStr".equals(name) || "v".equals(name)) {
vIsOpen = true;
// Clear contents cache
value.setLength(0);
}
// c => cell
else if ("c".equals(name)) {
// Get the cell reference
cellCoordinate = attributes.getValue("r");
int firstDigit = -1;
for (int c = 0; c < cellCoordinate.length(); ++c) {
if (Character.isDigit(cellCoordinate.charAt(c))) {
firstDigit = c;
break;
}
}
thisColumn = nameToColumn(cellCoordinate.substring(0, firstDigit));
// Set up defaults.
this.nextDataType = XssfDataType.NUMBER;
this.formatIndex = -1;
this.formatString = null;
String cellType = attributes.getValue("t");
String cellStyleStr = attributes.getValue("s");
if ("b".equals(cellType)) {
nextDataType = XssfDataType.BOOL;
} else if ("e".equals(cellType)) {
nextDataType = XssfDataType.ERROR;
} else if ("inlineStr".equals(cellType)) {
nextDataType = XssfDataType.INLINESTR;
} else if ("s".equals(cellType)) {
nextDataType = XssfDataType.SSTINDEX;
} else if ("str".equals(cellType)) {
nextDataType = XssfDataType.FORMULA;
} else if (cellStyleStr != null) {
// It's a number, but almost certainly one
// with a special style or format
int styleIndex = Integer.parseInt(cellStyleStr);
XSSFCellStyle style = stylesTable.getStyleAt(styleIndex);
this.formatIndex = style.getDataFormat();
this.formatString = style.getDataFormatString();
if (this.formatString == null) {
this.formatString = BuiltinFormats.getBuiltinFormat(this.formatIndex);
}
}
}
}
/*
* (non-Javadoc)
* #see org.xml.sax.helpers.DefaultHandler#endElement(java.lang.String, java.lang.String, java.lang.String)
*/
public void endElement(String uri, String localName, String name)
throws SAXException {
String cellValue = null;
//String thisStr = null;
// v => contents of a cell
if ("v".equals(name)) {
// Process the value contents as required.
// Do now, as characters() may be called more than once
switch (nextDataType) {
case BOOL:
char first = value.charAt(0);
//thisStr = first == '0' ? "FALSE" : "TRUE";
//cellValue= new Boolean(first =='0' ? false: true);
cellValue=first == '0' ? "false" : "true";
break;
case ERROR:
//thisStr = "\"ERROR:" + value.toString() + '"';
cellValue=new String(value.toString());
break;
case FORMULA:
// A formula could result in a string value,
// so always add double-quote characters.
//thisStr = '"' + value.toString() + '"';
cellValue=new String(value.toString());
break;
case INLINESTR:
XSSFRichTextString rtsi = new XSSFRichTextString(value.toString());
//thisStr = '"' + rtsi.toString() + '"';
cellValue=new String(rtsi.toString());
break;
case SSTINDEX:
String sstIndex = value.toString();
try {
int idx = Integer.parseInt(sstIndex);
XSSFRichTextString rtss = new XSSFRichTextString(sharedStringsTable.getEntryAt(idx));
// thisStr = '"' + rtss.toString() + '"';
cellValue=new String(rtss.toString());
}
catch (NumberFormatException ex) {
System.out.println("Failed to parse SST index '" + sstIndex + "': " + ex.toString());
}
break;
case NUMBER:
String n = value.toString();
if (this.formatString != null && n.length() > 0){
cellValue = formatter.formatRawCellContents(Double.parseDouble(n), this.formatIndex, this.formatString);
//cellValue=new Double(Double.parseDouble(n));
}
else{
//thisStr = n;
cellValue=new String(n);
}
break;
default:
cellValue="";
//thisStr = "(TODO: Unexpected type: " + nextDataType + ")";
break;
}
// Output after we've seen the string contents
// Emit commas for any fields that were missing on this row
if (lastColumnNumber == -1) {
lastColumnNumber = 0;
}
// for (int i = lastColumnNumber; i < thisColumn; ++i){
// System.out.print(',');
// }
// Might be the empty string.
rowValues.put(cellCoordinate,cellValue);
//System.out.print(cellValue);
// Update column
if (thisColumn > -1)
lastColumnNumber = thisColumn;
} else if ("row".equals(name)) {
// We're onto a new row
//I have to pass new HashMap because otherwise all the map into archiveAcquisition have the same values
databaseAcquisitionServices.archiveAcquisition(new TreeMap<>(rowValues), rowCounter, acquisitionForm, sheetIndex);
//Clear the structure used to store row data
rowValues.clear();
rowCounter++;
//System.out.println();
lastColumnNumber = -1;
}
}
/**
* Captures characters only if a suitable element is open.
* Originally was just "v"; extended for inlineStr also.
*/
public void characters(char[] ch, int start, int length)
throws SAXException {
if (vIsOpen)
value.append(ch, start, length);
}
/**
* Converts an Excel column name like "C" to a zero-based index.
*
* #param name
* #return Index corresponding to the specified name
*/
private int nameToColumn(String name) {
int column = -1;
for (int i = 0; i < name.length(); ++i) {
int c = name.charAt(i);
column = (column + 1) * 26 + c - 'A';
}
return column;
}

You will likely need to count the number of rows that you see in a class derived from SheetContentsHandler, I don't think you need to have a separate XSSFSheetHAndler, I would rather use the default one and only have a dervied SheetContentsHandler which has callback-methods for all the things that you are interested in, e.g. startRow, endRow, cell.
You can take a look at the XLSX2CSV sample for an example of how streaming reading of XLSX files can be done, especially how the class SheetToCSV is used to get calls for each row/cell, you could e.g. use something like if (currentRow < startRow) { return } in the method cell() to skip rows.
The cell-method is called with a parameter String cellReference which can be used to retrieve the row/cell coordinates via new CellReference(cellReference).getCol() and new CellReference(cellReference).getRow().

Apache POI WorkFactory.Create(new File()) java.lang.OutOfMemoryError

I'm trying to load an excel file(xlsx) into a Workbook Object using apache POI 3.10.
I'm receiving a java.lang.OutofMemoryError.
I'm using Java 8 with the -Xmx2g argument on the JVM.
All 4 cores(64bit System) and my RAM(4gb) are maxed out when I run the program.
The excel sheet has 43 columns and 166,961 Rows which equal 7,179,323 Cells.
I'm using Apache POIs WorkBookFactory.create(new File) because it uses less memory than using InputFileStream.
Does anyone have any ideas how to optimize memory usage or another way to create the Workbook?
Below is my test Reader class, don't judge, it's rough and includes debugging statements:
import java.io.File;
import java.io.IOException;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
public class Reader {
private Workbook wb;
public Reader(File excel) {
System.out.println("CONSTRUCTOR");
wb = null;
try {
wb = WorkbookFactory.create(excel);
} catch (IOException e) {
System.out.println("IO Exception");
System.out.println(e.getMessage());
} catch (InvalidFormatException e) {
System.out.println("Invalid Format");
System.out.println(e.getMessage());
}
}
public boolean exists() { return (wb != null); }
public void print() {}
public static void main(String[] args) {
System.out.println("START PRG");
//File f = new File("oldfilename.xls");
File f = new File("filename.xlsx");
System.out.println("PATH:" + f.getAbsoluteFile());
if (!f.exists()) {
System.out.println("File does not exist.");
System.exit(0);
}
System.out.println("FILE");
Reader r = new Reader(f);
System.out.println("Reader");
r.print();
System.out.println("PRG DONE");
}
}

apparently loading a 24mb file shouldn't be causing OOM...
at first glance it appears to me, though Xmx set to 2G, there's actually not that much memory free in system. in other words OS and other processes may have taken more than 2G out of 4G of physical memory! Check available physical memory first. in case available below what's expected, try closing some other running apps/processes.
if that's not the case and there's indeed enough memory left, without profiling it's really hard to identify the real cause. use a profile tool to check JVM status, related to memory first. you may simply use jconsole (as it comes with JDK). #see this on how to activate JMX
once you are connected, check readings related to memory, specifically below memory spaces:
old gen
young gen
perm gen
monitor these spaces and see where it's struggling. I assume this is a standalone application. in case this is deployed on server (as web or services), you may consider '-XX:NewRatio' option for distributing heap spaces effectively and efficiently. #see tuning related details here.

Please confirm these before proceeding,
Is there any infinite execution in looping(for/while)
Ensure your physical storage size
Maximize buffer memory
Note
As per my understanding Apache POI will not consume that much amount of memory.

I am just a beginner, but may I ask you some questions.
Why not use XSSFWorkbook class to open XLSX file. I mean, I always use it to handle XLSX files, and this time I tried with a file(7 MB; that was the largest I could find in my computer), and it worked perfectly.
Why not use newer File API(NIO, Java 7). Again, I do not know if this will make any difference or not. But, it worked for me.
Windows 7 Ultimate | 64 bit | Intel 2nd Gen Core i3|Eclipse Juno|JDK 1.7.45|Apache POI 3.9
Path file = Paths.get("XYZABC.xlsx");
try {
XSSFWorkbook wb = new XSSFWorkbook(Files.newInputStream(file, StandardOpenOption.READ));
} catch (IOException e) {
System.out.println("Some IO Error!!!!");
}
Do, tell if it works for you or not.

Did you tried using SXSSFWorkbook? We also used Apache POI to handle relatively big XLSX files, and we also had memory problems when using plain XSSFWorkbook. Although we didn't have to read in the files, we were just writing tens of thousands of lines of informations. Using this, our memory problems got solved. You can pass an XSSFWorkbook to its constructor and the size of data you want to keep in memory.

Java 1.8
based on HSSF and XSSF Limitations
my poi version is 3.17 POI Examples
lauches my code
public class Controller {
EX stressTest;
public void fineFile() {
String stresstest = "C:\\Stresstest.xlsx";
HashMap<String, String[]> stressTestMap = new HashMap<>();
stressTestMap.put("aaaa", new String[]{"myField", "The field"});
stressTestMap.put("bbbb", new String[]{"other", "Other value"});
try {
InputStream stressTestIS = new FileInputStream(stresstest);
stressTest = new EX(stresstest, stressTestIS, stressTestMap);
} catch (IOException exp) {
}
}
public void printErr() {
if (stressTest.thereAreErrors()) {
try {
FileWriter myWriter = new FileWriter(
"C:\\logErrorsStressTest" +
(new SimpleDateFormat("ddMMyyyyHHmmss")).format(new Date()) +
".txt"
);
myWriter.write(stressTest.getBodyFileErrors());
myWriter.close();
} catch (IOException e) {
e.printStackTrace();
}
} else {
}
}
public void createBD() {
List<OneObjectWhatever> entitiesList =
(
!stressTest.thereAreErrors()
? ((List<OneObjectWhatever>) stressTest.toListCustomerObject(OneObjectWhatever.class))
: new ArrayList<>()
);
entitiesList.forEach(entity -> {
Field[] fields = entity.getClass().getDeclaredFields();
String valueString = "";
for (Field attr : fields) {
try {
attr.setAccessible(true);
valueString += " StressTest:" + attr.getName() + ": -" + attr.get(fields) + "- ";
attr.setAccessible(true);
} catch (Exception reflectionError) {
System.out.println(reflectionError);
}
}
});
}
}
MY CODE
public class EX {
private HashMap<Integer, HashMap<Integer, String> > rows;
private List<String> errors;
private int maxColOfHeader, minColOfHeader;
private HashMap<Integer, String> header;
private HashMap<String,String[]> relationHeaderClassPropertyDescription;
private void initVariables(String name, InputStream file) {
this.rows = new HashMap();
this.header = new HashMap<>();
this.errors = new ArrayList<String>(){{add("["+name+"] empty cells in position -> ");}};
try{
InputStream is = FileMagic.prepareToCheckMagic(file);
FileMagic fm = FileMagic.valueOf(is);
is.close();
switch (fm) {
case OLE2:
XLS2CSVmra xls2csv = new XLS2CSVmra(name, 50, rows);
xls2csv.process();
System.out.println("OLE2");
break;
case OOXML:
File flatFile = new File(name);
OPCPackage p = OPCPackage.open(flatFile, PackageAccess.READ);
XLSX2CSV xlsx2csv = new XLSX2CSV(p, System.out, 50, this.rows);
xlsx2csv.process();
p.close();
System.out.println("OOXML");
break;
default:
System.out.println("Your InputStream was neither an OLE2 stream, nor an OOXML stream");
break;
}
} catch (IOException | EncryptedDocumentException | SAXException | OpenXML4JException exp){
System.out.println(exp);
exp.printStackTrace();
}
int rowHeader = rows.keySet().stream().findFirst().get();
this.header.putAll(rows.get(rowHeader));
this.rows.remove(rowHeader);
this.minColOfHeader = this.header.keySet().stream().findFirst().get();
this.maxColOfHeader = this.header.entrySet().stream()
.mapToInt(e -> e.getKey()).max()
.orElseThrow(NoSuchElementException::new);
}
public EX(String name, InputStream file, HashMap<String,String[]> relationHeaderClassPropertyDescription_) {
this.relationHeaderClassPropertyDescription = relationHeaderClassPropertyDescription_;
initVariables(name, file);
validate();
}
private void validate(){
rows.forEach((inx,row) -> {
for(int i = minColOfHeader; i <= maxColOfHeader; i++) {
//System.out.println("r:"+inx+" c:"+i+" cr:"+(!row.containsKey(i))+" vr:"+((!row.containsKey(i)) || row.get(i).trim().isEmpty())+" ch:"+header.containsKey(i)+" vh:"+(header.containsKey(i) && (!header.get(i).trim().isEmpty()))+" val:"+(row.containsKey(i)&&!row.get(i).trim().isEmpty()?row.get(i):"empty"));
if((!row.containsKey(i)) || row.get(i).trim().isEmpty()) {
if(header.containsKey(i) && (!header.get(i).trim().isEmpty())) {
String description = getRelationHeaders(i,1);
errors.add(" ["+header.get(i)+"]{"+description+"} = fila: "+(inx+1)+" - columna: "+ CellReference.convertNumToColString(i));
// System.out.println(" fila: "+inx+" - columna: " + i + " - valor: "+ (row.get(i).isEmpty()?"empty":row.get(i)));
}
}
}
});
header.forEach((i,v)->{System.out.println("stressTestMap.put(\""+v+"\", new String[]{\"{"+i+"}\",\"Mi descripcion XD\"});");});
}
public String getBodyFileErrors()
{
return String.join(System.lineSeparator(), errors);
}
public boolean thereAreErrors() {
return errors.stream().count() > 1;
}
public<T extends Class> List<? extends Object> toListCustomerObject(T type) {
List<Object> list = new ArrayList<>();
rows.forEach((inx, row) -> {
try {
Object obj = type.newInstance();
for(int i = minColOfHeader; i <= maxColOfHeader; i++) {
if (row.containsKey(i) && !row.get(i).trim().isEmpty()) {
if (header.containsKey(i) && !header.get(i).trim().isEmpty()) {
if(relationHeaderClassPropertyDescription.containsKey(header.get(i))) {
String nameProperty = getRelationHeaders(i,0);
Field field = type.getDeclaredField(nameProperty);
try{
field.setAccessible(true);
field.set(obj, (isConvertibleTo(field.getType(),row.get(i)) ? toObject(field.getType(),row.get(i)) : defaultValue(field.getType())) );
field.setAccessible(false);
}catch (Exception fex) {
//System.out.println("113"+fex);
continue;
}
}
}
}
}
list.add(obj);
} catch (Exception ex) {
//System.out.println("123:"+ex);
}
});
return list;
}
private Object toObject( Class clazz, String value ) {
if( Boolean.class == clazz || Boolean.TYPE == clazz) return Boolean.parseBoolean( value );
if( Byte.class == clazz || Byte.TYPE == clazz) return Byte.parseByte( value );
if( Short.class == clazz || Short.TYPE == clazz) return Short.parseShort( value );
if( Integer.class == clazz || Integer.TYPE == clazz) return Integer.parseInt( value );
if( Long.class == clazz || Long.TYPE == clazz) return Long.parseLong( value );
if( Float.class == clazz || Float.TYPE == clazz) return Float.parseFloat( value );
if( Double.class == clazz || Double.TYPE == clazz) return Double.parseDouble( value );
return value;
}
private boolean isConvertibleTo( Class clazz, String value ) {
String ptn = "";
if( Boolean.class == clazz || Boolean.TYPE == clazz) ptn = ".*";
if( Byte.class == clazz || Byte.TYPE == clazz) ptn = "^\\d+$";
if( Short.class == clazz || Short.TYPE == clazz) ptn = "^\\d+$";
if( Integer.class == clazz || Integer.TYPE == clazz) ptn = "^\\d+$";
if( Long.class == clazz || Long.TYPE == clazz) ptn = "^\\d+$";
if( Float.class == clazz || Float.TYPE == clazz) ptn = "^\\d+(\\.\\d+)?$";
if( Double.class == clazz || Double.TYPE == clazz) ptn = "^\\d+(\\.\\d+)?$";
Pattern pattern = Pattern.compile(ptn, Pattern.CASE_INSENSITIVE);
Matcher matcher = pattern.matcher(value);
return matcher.find();
}
private Object defaultValue( Class clazz) {
if( Boolean.class == clazz || Boolean.TYPE == clazz) return Boolean.parseBoolean( "false" );
if( Byte.class == clazz || Byte.TYPE == clazz) return Byte.parseByte( "0" );
if( Short.class == clazz || Short.TYPE == clazz) return Short.parseShort( "0" );
if( Integer.class == clazz || Integer.TYPE == clazz) return Integer.parseInt( "0" );
if( Long.class == clazz || Long.TYPE == clazz) return Long.parseLong( "0" );
if( Float.class == clazz || Float.TYPE == clazz) return Float.parseFloat( "0.0" );
if( Double.class == clazz || Double.TYPE == clazz) return Double.parseDouble( "0.0" );
return "";
}
private String getRelationHeaders(Integer columnIndexHeader, Integer TypeOrDescription /*0 - Type, 1 - Description*/) {
try {
return relationHeaderClassPropertyDescription.get(header.get(columnIndexHeader))[TypeOrDescription];
} catch (Exception e) {
}
return header.get(columnIndexHeader);
}
}
these are the modifications I made to the examples:
XLSX2CSV
public class XLSX2CSV {
/**
* Uses the XSSF Event SAX helpers to do most of the work
* of parsing the Sheet XML, and outputs the contents
* as a (basic) CSV.
*/
private class SheetToCSV implements SheetContentsHandler {
private boolean firstCellOfRow = false;
private int currentRow = -1;
private int currentCol = -1;
HashMap<Integer, String> valuesCell;
private void outputMissingRows(int number) {
for (int i=0; i<number; i++) {
for (int j=0; j<minColumns; j++) {
output.append(',');
}
output.append('\n');
}
}
#Override
public void startRow(int rowNum) {
// If there were gaps, output the missing rows
outputMissingRows(rowNum-currentRow-1);
// Prepare for this row
firstCellOfRow = true;
currentRow = rowNum;
currentCol = -1;
valuesCell = new HashMap<>();
}
#Override
public void endRow(int rowNum) {
// Ensure the minimum number of columns
for (int i = currentCol; i < minColumns; i++) {
output.append(',');
}
output.append('\n');
if (!valuesCell.isEmpty())
_rows.put(rowNum, valuesCell);
}
#Override
public void cell(String cellReference, String formattedValue,
XSSFComment comment) {
if (firstCellOfRow) {
firstCellOfRow = false;
} else {
output.append(',');
}
// gracefully handle missing CellRef here in a similar way as XSSFCell does
if (cellReference == null) {
cellReference = new CellAddress(currentRow, currentCol).formatAsString();
}
// Did we miss any cells?
int thisCol = (new CellReference(cellReference)).getCol();
int missedCols = thisCol - currentCol - 1;
for (int i = 0; i < missedCols; i++) {
output.append(',');
}
currentCol = thisCol;
if (!formattedValue.isEmpty())
valuesCell.put(thisCol, formattedValue);
// Number or string?
output.append(formattedValue);
/*try {
//noinspection ResultOfMethodCallIgnored
Double.parseDouble(formattedValue);
output.append(formattedValue);
} catch (NumberFormatException e) {
output.append('"');
output.append(formattedValue);
output.append('"');
}*/
}
#Override
public void headerFooter(String text, boolean isHeader, String tagName) {
// Skip, no headers or footers in CSV
}
}
///////////////////////////////////////
private final OPCPackage xlsxPackage;
/**
* Number of columns to read starting with leftmost
*/
private final int minColumns;
/**
* Destination for data
*/
private final PrintStream output;
public HashMap<Integer, HashMap<Integer, String>> _rows;
/**
* Creates a new XLSX -> CSV converter
*
* #param pkg The XLSX package to process
* #param output The PrintStream to output the CSV to
* #param minColumns The minimum number of columns to output, or -1 for no minimum
*/
public XLSX2CSV(OPCPackage pkg, PrintStream output, int minColumns, HashMap<Integer, HashMap<Integer, String> > __rows) {
this.xlsxPackage = pkg;
this.output = output;
this.minColumns = minColumns;
this._rows = __rows;
}
/**
* Parses and shows the content of one sheet
* using the specified styles and shared-strings tables.
*
* #param styles The table of styles that may be referenced by cells in the sheet
* #param strings The table of strings that may be referenced by cells in the sheet
* #param sheetInputStream The stream to read the sheet-data from.
* #exception java.io.IOException An IO exception from the parser,
* possibly from a byte stream or character stream
* supplied by the application.
* #throws SAXException if parsing the XML data fails.
*/
public void processSheet(
StylesTable styles,
ReadOnlySharedStringsTable strings,
SheetContentsHandler sheetHandler,
InputStream sheetInputStream) throws IOException, SAXException {
DataFormatter formatter = new DataFormatter();
InputSource sheetSource = new InputSource(sheetInputStream);
try {
XMLReader sheetParser = SAXHelper.newXMLReader();
ContentHandler handler = new XSSFSheetXMLHandler(
styles, null, strings, sheetHandler, formatter, false);
sheetParser.setContentHandler(handler);
sheetParser.parse(sheetSource);
} catch(ParserConfigurationException e) {
throw new RuntimeException("SAX parser appears to be broken - " + e.getMessage());
}
}
/**
* Initiates the processing of the XLS workbook file to CSV.
*
* #throws IOException If reading the data from the package fails.
* #throws SAXException if parsing the XML data fails.
*/
public void process() throws IOException, OpenXML4JException, SAXException {
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(this.xlsxPackage);
XSSFReader xssfReader = new XSSFReader(this.xlsxPackage);
StylesTable styles = xssfReader.getStylesTable();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData();
int index = 0;
while (iter.hasNext()) {
InputStream stream = iter.next();
String sheetName = iter.getSheetName();
this.output.println();
this.output.println(sheetName + " [index=" + index + "]:");
processSheet(styles, strings, new SheetToCSV(), stream);
stream.close();
++index;
break;
}
}
}
XLS2CSVmra
public class XLS2CSVmra implements HSSFListener {
private int minColumns;
private POIFSFileSystem fs;
private PrintStream output;
public HashMap<Integer, HashMap<Integer, String>> _rows;
private HashMap<Integer, String> valuesCell;
private int lastRowNumber;
private int lastColumnNumber;
/** Should we output the formula, or the value it has? */
private boolean outputFormulaValues = false;
/** For parsing Formulas */
private SheetRecordCollectingListener workbookBuildingListener;
private HSSFWorkbook stubWorkbook;
// Records we pick up as we process
private SSTRecord sstRecord;
private FormatTrackingHSSFListener formatListener;
/** So we known which sheet we're on */
private int sheetIndex = -1;
private BoundSheetRecord[] orderedBSRs;
private List<BoundSheetRecord> boundSheetRecords = new ArrayList<BoundSheetRecord>();
// For handling formulas with string results
private int nextRow;
private int nextColumn;
private boolean outputNextStringRecord;
/**
* Creates a new XLS -> CSV converter
* #param fs The POIFSFileSystem to process
* #param output The PrintStream to output the CSV to
* #param minColumns The minimum number of columns to output, or -1 for no minimum
*/
public XLS2CSVmra(POIFSFileSystem fs, PrintStream output, int minColumns, HashMap<Integer, HashMap<Integer, String>> __rows) {
this.fs = fs;
this.output = output;
this.minColumns = minColumns;
this._rows = __rows;
this.valuesCell = new HashMap<>();
}
/**
* Creates a new XLS -> CSV converter
* #param filename The file to process
* #param minColumns The minimum number of columns to output, or -1 for no minimum
* #throws IOException
* #throws FileNotFoundException
*/
public XLS2CSVmra(String filename, int minColumns, HashMap<Integer, HashMap<Integer, String>> __rows) throws IOException, FileNotFoundException {
this(
new POIFSFileSystem(new FileInputStream(filename)),
System.out, minColumns,
__rows
);
}
/**
* Initiates the processing of the XLS file to CSV
*/
public void process() throws IOException {
MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this);
formatListener = new FormatTrackingHSSFListener(listener);
HSSFEventFactory factory = new HSSFEventFactory();
HSSFRequest request = new HSSFRequest();
if(outputFormulaValues) {
request.addListenerForAllRecords(formatListener);
} else {
workbookBuildingListener = new SheetRecordCollectingListener(formatListener);
request.addListenerForAllRecords(workbookBuildingListener);
}
factory.processWorkbookEvents(request, fs);
}
/**
* Main HSSFListener method, processes events, and outputs the
* CSV as the file is processed.
*/
#Override
public void processRecord(Record record) {
if(sheetIndex>0)
return;
int thisRow = -1;
int thisColumn = -1;
String thisStr = null;
switch (record.getSid())
{
case BoundSheetRecord.sid:
if(sheetIndex==-1)
boundSheetRecords.add((BoundSheetRecord)record);
break;
case BOFRecord.sid:
BOFRecord br = (BOFRecord)record;
if(br.getType() == BOFRecord.TYPE_WORKSHEET && sheetIndex==-1) {
// Create sub workbook if required
if(workbookBuildingListener != null && stubWorkbook == null) {
stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook();
}
// Output the worksheet name
// Works by ordering the BSRs by the location of
// their BOFRecords, and then knowing that we
// process BOFRecords in byte offset order
sheetIndex++;
if(orderedBSRs == null) {
orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords);
}
output.println();
output.println(
orderedBSRs[sheetIndex].getSheetname() +
" [" + (sheetIndex+1) + "]:"
);
}
break;
case SSTRecord.sid:
sstRecord = (SSTRecord) record;
break;
case BlankRecord.sid:
BlankRecord brec = (BlankRecord) record;
thisRow = brec.getRow();
thisColumn = brec.getColumn();
thisStr = "";
break;
case BoolErrRecord.sid:
BoolErrRecord berec = (BoolErrRecord) record;
thisRow = berec.getRow();
thisColumn = berec.getColumn();
thisStr = "";
break;
case FormulaRecord.sid:
FormulaRecord frec = (FormulaRecord) record;
thisRow = frec.getRow();
thisColumn = frec.getColumn();
if(outputFormulaValues) {
if(Double.isNaN( frec.getValue() )) {
// Formula result is a string
// This is stored in the next record
outputNextStringRecord = true;
nextRow = frec.getRow();
nextColumn = frec.getColumn();
} else {
thisStr = formatListener.formatNumberDateCell(frec);
}
} else {
thisStr = '"' +
HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"';
}
break;
case StringRecord.sid:
if(outputNextStringRecord) {
// String for formula
StringRecord srec = (StringRecord)record;
thisStr = srec.getString();
thisRow = nextRow;
thisColumn = nextColumn;
outputNextStringRecord = false;
}
break;
case LabelRecord.sid:
LabelRecord lrec = (LabelRecord) record;
thisRow = lrec.getRow();
thisColumn = lrec.getColumn();
thisStr = '"' + lrec.getValue() + '"';
break;
case LabelSSTRecord.sid:
LabelSSTRecord lsrec = (LabelSSTRecord) record;
thisRow = lsrec.getRow();
thisColumn = lsrec.getColumn();
if(sstRecord == null) {
thisStr = '"' + "(No SST Record, can't identify string)" + '"';
} else {
thisStr = '"' + sstRecord.getString(lsrec.getSSTIndex()).toString() + '"';
}
break;
case NoteRecord.sid:
NoteRecord nrec = (NoteRecord) record;
thisRow = nrec.getRow();
thisColumn = nrec.getColumn();
// TODO: Find object to match nrec.getShapeId()
thisStr = '"' + "(TODO)" + '"';
break;
case NumberRecord.sid:
NumberRecord numrec = (NumberRecord) record;
thisRow = numrec.getRow();
thisColumn = numrec.getColumn();
// Format
thisStr = formatListener.formatNumberDateCell(numrec);
break;
case RKRecord.sid:
RKRecord rkrec = (RKRecord) record;
thisRow = rkrec.getRow();
thisColumn = rkrec.getColumn();
thisStr = '"' + "(TODO)" + '"';
break;
default:
break;
}
// Handle new row
if(thisRow != -1 && thisRow != lastRowNumber) {
lastColumnNumber = -1;
}
// Handle missing column
if(record instanceof MissingCellDummyRecord) {
MissingCellDummyRecord mc = (MissingCellDummyRecord)record;
thisRow = mc.getRow();
thisColumn = mc.getColumn();
thisStr = "";
}
// If we got something to print out, do so
if(thisStr != null) {
if (thisColumn > 0) {
output.print(',');
}
if (!thisStr.isEmpty())
valuesCell.put(thisColumn, thisStr);
output.print(thisStr);
}
// Update column and row count
if(thisRow > -1)
lastRowNumber = thisRow;
if(thisColumn > -1)
lastColumnNumber = thisColumn;
// Handle end of row
if(record instanceof LastCellOfRowDummyRecord) {
// Print out any missing commas if needed
if(minColumns > 0) {
// Columns are 0 based
if(lastColumnNumber == -1) { lastColumnNumber = 0; }
for(int i=lastColumnNumber; i<(minColumns); i++) {
output.print(',');
}
}
// We're onto a new row
lastColumnNumber = -1;
// End the row
output.println();
if(!valuesCell.isEmpty()) {
HashMap<Integer, String> newRow = new HashMap<>();
valuesCell.forEach((inx,vStr) -> {
newRow.put(inx, vStr);
});
_rows.put(lastRowNumber, newRow);
valuesCell = new HashMap<>();
}
}
}
}

Reading an Excel sheet using POI's XSSF and SAX (Event API)

I am reading an Excel sheet using POI's XSSF and SAX (Event API). The Excel sheet has thousands of rows of user information like user name, email, address, age, department etc.
I need to read each row from Excel, convert it into a User object and add this User object to a List of User objects.
I can read the Excel sheet successfully, but I am not sure at what point while reading I should create an instance of the User object and populate it with the data from the Excel sheet.
Below is my entire working code.
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.opc.PackageAccess;
import org.apache.poi.ss.usermodel.BuiltinFormats;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.model.StylesTable;
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
public class ExcelSheetParser {
enum xssfDataType {
BOOL, ERROR, FORMULA, INLINESTR, SSTINDEX, NUMBER,
}
int countrows = 0;
class XSSFSheetHandler extends DefaultHandler {
/**
* Table with styles
*/
private StylesTable stylesTable;
/**
* Table with unique strings
*/
private ReadOnlySharedStringsTable sharedStringsTable;
/**
* Destination for data
*/
private final PrintStream output;
private List<?> list = new ArrayList();
private Class clazz;
/**
* Number of columns to read starting with leftmost
*/
private final int minColumnCount;
// Set when V start element is seen
private boolean vIsOpen;
// Set when cell start element is seen;
// used when cell close element is seen.
private xssfDataType nextDataType;
// Used to format numeric cell values.
private short formatIndex;
private String formatString;
private final DataFormatter formatter;
private int thisColumn = -1;
// The last column printed to the output stream
private int lastColumnNumber = -1;
// Gathers characters as they are seen.
private StringBuffer value;
/**
* Accepts objects needed while parsing.
*
* #param styles
* Table of styles
* #param strings
* Table of shared strings
* #param cols
* Minimum number of columns to show
* #param target
* Sink for output
*/
public XSSFSheetHandler(StylesTable styles,
ReadOnlySharedStringsTable strings, int cols, PrintStream target, Class clazz) {
this.stylesTable = styles;
this.sharedStringsTable = strings;
this.minColumnCount = cols;
this.output = target;
this.value = new StringBuffer();
this.nextDataType = xssfDataType.NUMBER;
this.formatter = new DataFormatter();
this.clazz = clazz;
}
public void startElement(String uri, String localName, String name,
Attributes attributes) throws SAXException {
if ("inlineStr".equals(name) || "v".equals(name)) {
vIsOpen = true;
// Clear contents cache
value.setLength(0);
}
// c => cell
else if ("c".equals(name)) {
// Get the cell reference
String r = attributes.getValue("r");
int firstDigit = -1;
for (int c = 0; c < r.length(); ++c) {
if (Character.isDigit(r.charAt(c))) {
firstDigit = c;
break;
}
}
thisColumn = nameToColumn(r.substring(0, firstDigit));
// Set up defaults.
this.nextDataType = xssfDataType.NUMBER;
this.formatIndex = -1;
this.formatString = null;
String cellType = attributes.getValue("t");
String cellStyleStr = attributes.getValue("s");
if ("b".equals(cellType))
nextDataType = xssfDataType.BOOL;
else if ("e".equals(cellType))
nextDataType = xssfDataType.ERROR;
else if ("inlineStr".equals(cellType))
nextDataType = xssfDataType.INLINESTR;
else if ("s".equals(cellType))
nextDataType = xssfDataType.SSTINDEX;
else if ("str".equals(cellType))
nextDataType = xssfDataType.FORMULA;
else if (cellStyleStr != null) {
// It's a number, but almost certainly one
// with a special style or format
int styleIndex = Integer.parseInt(cellStyleStr);
XSSFCellStyle style = stylesTable.getStyleAt(styleIndex);
this.formatIndex = style.getDataFormat();
this.formatString = style.getDataFormatString();
if (this.formatString == null)
this.formatString = BuiltinFormats
.getBuiltinFormat(this.formatIndex);
}
}
}
public void endElement(String uri, String localName, String name)
throws SAXException {
String thisStr = null;
// v => contents of a cell
if ("v".equals(name)) {
// Process the value contents as required.
// Do now, as characters() may be called more than once
switch (nextDataType) {
case BOOL:
char first = value.charAt(0);
thisStr = first == '0' ? "FALSE" : "TRUE";
break;
case ERROR:
thisStr = "\"ERROR:" + value.toString() + '"';
break;
case FORMULA:
// A formula could result in a string value,
// so always add double-quote characters.
thisStr = '"' + value.toString() + '"';
break;
case INLINESTR:
// TODO: have seen an example of this, so it's untested.
XSSFRichTextString rtsi = new XSSFRichTextString(value
.toString());
thisStr = '"' + rtsi.toString() + '"';
break;
case SSTINDEX:
String sstIndex = value.toString();
try {
int idx = Integer.parseInt(sstIndex);
XSSFRichTextString rtss = new XSSFRichTextString(
sharedStringsTable.getEntryAt(idx));
thisStr = '"' + rtss.toString() + '"';
} catch (NumberFormatException ex) {
output.println("Failed to parse SST index '" + sstIndex
+ "': " + ex.toString());
}
break;
case NUMBER:
String n = value.toString();
if (this.formatString != null)
thisStr = formatter.formatRawCellContents(Double
.parseDouble(n), this.formatIndex,
this.formatString);
else
thisStr = n;
break;
default:
thisStr = "(TODO: Unexpected type: " + nextDataType + ")";
break;
}
// Output after we've seen the string contents
// Emit commas for any fields that were missing on this row
if (lastColumnNumber == -1) {
lastColumnNumber = 0;
}
for (int i = lastColumnNumber; i < thisColumn; ++i)
output.print(',');
// Might be the empty string.
output.print(thisColumn +" : "+thisStr);
// Update column
if (thisColumn > -1)
lastColumnNumber = thisColumn;
} else if ("row".equals(name)) {
// Print out any missing commas if needed
if (minColumns > 0) {
// Columns are 0 based
if (lastColumnNumber == -1) {
lastColumnNumber = 0;
}
for (int i = lastColumnNumber; i < (this.minColumnCount); i++) {
output.print(',');
}
}
// We're onto a new row
output.println();
output.println(countrows++);
lastColumnNumber = -1;
}
}
/**
* Captures characters only if a suitable element is open. Originally
* was just "v"; extended for inlineStr also.
*/
public void characters(char[] ch, int start, int length)
throws SAXException {
if (vIsOpen)
value.append(ch, start, length);
}
/**
* Converts an Excel column name like "C" to a zero-based index.
*
* #param name
* #return Index corresponding to the specified name
*/
private int nameToColumn(String name) {
int column = -1;
for (int i = 0; i < name.length(); ++i) {
int c = name.charAt(i);
column = (column + 1) * 26 + c - 'A';
}
return column;
}
}
// /////////////////////////////////////
private OPCPackage xlsxPackage;
private int minColumns;
private PrintStream output;
private Class clazz;
/**
* Creates a new XLSX -> CSV converter
*
* #param pkg
* The XLSX package to process
* #param output
* The PrintStream to output the CSV to
* #param minColumns
* The minimum number of columns to output, or -1 for no minimum
*/
public ExcelSheetParser(OPCPackage pkg, PrintStream output, int minColumns, Class clazz) {
this.xlsxPackage = pkg;
this.output = output;
this.minColumns = minColumns;
this.clazz = clazz;
}
/**
* Parses and shows the content of one sheet using the specified styles and
* shared-strings tables.
*
* #param styles
* #param strings
* #param sheetInputStream
*/
public void processSheet(StylesTable styles,
ReadOnlySharedStringsTable strings, InputStream sheetInputStream)
throws IOException, ParserConfigurationException, SAXException {
InputSource sheetSource = new InputSource(sheetInputStream);
SAXParserFactory saxFactory = SAXParserFactory.newInstance();
SAXParser saxParser = saxFactory.newSAXParser();
XMLReader sheetParser = saxParser.getXMLReader();
ContentHandler handler = new XSSFSheetHandler(styles, strings,
this.minColumns, this.output, this.clazz);
sheetParser.setContentHandler(handler);
sheetParser.parse(sheetSource);
}
/**
* Initiates the processing of the XLS workbook file to CSV.
*
* #throws IOException
* #throws OpenXML4JException
* #throws ParserConfigurationException
* #throws SAXException
*/
public void process() throws IOException, OpenXML4JException,
ParserConfigurationException, SAXException {
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(
this.xlsxPackage);
XSSFReader xssfReader = new XSSFReader(this.xlsxPackage);
StylesTable styles = xssfReader.getStylesTable();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader
.getSheetsData();
int index = 0;
while (iter.hasNext()) {
InputStream stream = iter.next();
String sheetName = iter.getSheetName();
this.output.println(sheetName + " [index=" + index + "]:");
processSheet(styles, strings, stream);
stream.close();
++index;
}
}
}

What I'd probably do is start building the User object when the row starts. As you hit the cells in the row, you populate your User object. When the row ends, validate the User object, and if it's fine add it then. Because you're doing SAX parsing, you'll get the start and events for all of these, so you can attach your logic there.
I'd suggest you take a look at XLSX2CSV in the Apache POI Examples. It shows how to go about handling the different kinds of cell contents (which you'll need for populating your user object), how to do something when you reach the end of the row, as well as handling missing cells etc.

I think you can create a user object at following location in your code:
// We're onto a new row
output.println();
// Convert output to a new user object
// ....
// ....

First of all where you are saving value in thisStr variable, if this is a valid value then put this value in Map.
You should create USer object in endElement() method in
else if ("row".equals(name)) {
// use map create USER object here
}
and You can add Users object in global list and if you want to persist it then you can persist it sheet by sheet OR all data at a time.
while (iter.hasNext()) {
InputStream stream = iter.next();
String sheetName = iter.getSheetName();
this.output.println(sheetName + " [index=" + index + "]:");
processSheet(styles, strings, stream);
stream.close();
++index;
//for persisting USERS data sheet by sheet write your code here.........
}
// for persisting complete data of all sheets write your code here...
This is working for me.