Add files via upload

Author: nsquare-jdzone

IndexCSVFilters.java

@@ -0,0 +1,131 @@
+packagecom.solr.index;
+
+importorg.apache.solr.client.solrj.SolrClient;
+importorg.apache.solr.client.solrj.SolrServerException;
+importorg.apache.solr.client.solrj.impl.HttpSolrClient;
+importorg.apache.solr.common.SolrInputDocument;
+importorg.slf4j.Logger;
+importorg.slf4j.LoggerFactory;
+
+importjava.io.*;
+importjava.nio.charset.StandardCharsets;
+importjava.text.ParseException;
+importjava.text.SimpleDateFormat;
+importjava.util.ArrayList;
+importjava.util.Collection;
+importjava.util.TimeZone;
+
+publicclassIndexCSVFilters{
+
+privateSolrClientsolrClient;
+privatestaticfinalTimeZoneUTC = TimeZone.getTimeZone("UTC");
+privateSimpleDateFormatsolrCompatibleSdf,csvCompatibleSdf;
+privatestaticfinalLoggerlogger = LoggerFactory.getLogger(IndexCSVFilters.class);
+
+privatestaticfinalStringTIME_STAMP = "T00:00:00Z";
+privatestaticfinalStringDATE_STAMP = "1970-01-01T";
+privatestaticfinalStringDATE_TIME_FIELD = "DateTime";
+privatestaticfinalStringTIME_FIELD = "Time";
+privatestaticfinalStringID_FIELD = "id";
+privatestaticfinalStringDATE_PART_Z = "Z";
+// private static final String DATA_SET_NAME_FIELD = "DataSetName"
+publicIndexCSVFilters(StringsolrUrl, StringcoreName){
+StringurlString = solrUrl+"/"+coreName;
+solrClient = newHttpSolrClient.Builder(urlString)
+ .withSocketTimeout(0)
+ .withConnectionTimeout(0)
+ .build();
+solrCompatibleSdf = newSimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss'Z'");
+solrCompatibleSdf.setTimeZone(UTC);
+csvCompatibleSdf = newSimpleDateFormat("MM/dd/yyyy hh:mm:ss aa");
+csvCompatibleSdf.setTimeZone(UTC);
+ }
+publicbooleanindexCSVFile(Filefile){
+
+logger.info("Indexing Started for file : "+file.getName());
+try(BufferedReaderbufferedReader = newBufferedReader(newInputStreamReader(newFileInputStream(file), StandardCharsets.UTF_8))){
+
+Stringline = null;
+line = bufferedReader.readLine();
+Stringsep = "";
+if(line.contains(",")){
+sep = ",";
+ }
+StringfieldNames[] = line.split(sep);
+Collection<SolrInputDocument> docList = newArrayList<>();
+// DataSetName,TimeStamp,First,Max,Min,Last,ValB,ValA,FilterS,FilterE,FilterT,ValS,ValC
+// E8374H231J#Type0,2017-01-09 09:31:00,66.2,71,66.2,71,66.2,71,35,2017-01-13,C,103.42,0
+// E8374H231J#Type0,2017-01-09 09:32:00,66.2,71,66.2,71,66.2,71,35,2017-01-13,C,103.57,0
+intcnt =0;
+intlineCnt =0;
+while ((line = bufferedReader.readLine())!=null){
+lineCnt++;
+if(line.trim().length()==0){
+continue;
+ }
+Stringdata [] = line.trim().split(sep);
+if(data.length!=fieldNames.length){
+System.out.println(lineCnt+"\t"+line);
+continue;
+ }
+SolrInputDocumentsolrInputDocument = newSolrInputDocument();
+for(intindex = 0;index<fieldNames.length;index++){
+
+if(index==1 || index==9){
+continue;
+ }
+solrInputDocument.addField(fieldNames[index],data[index]);
+ }
+Stringtime = data[1].substring(data[1].indexOf(" ")+1);
+
+solrInputDocument.addField(TIME_FIELD,DATE_STAMP+time+ DATE_PART_Z);
+solrInputDocument.addField(DATE_TIME_FIELD,data[1].replace(" ","T")+ DATE_PART_Z);
+solrInputDocument.addField(fieldNames[9],data[9]+TIME_STAMP);
+solrInputDocument.addField(ID_FIELD,data[1].replace(" ","_")+"_"+data[0]+"_"+data[8]+"_"+data[9]+"_"+data[10]);
+docList.add(solrInputDocument);
+
+if(cnt==100000){
+cnt = 0;
+solrClient.add(docList);
+solrClient.commit();
+docList.clear();
+ }else{
+cnt++;
+ }
+ }
+if(cnt!=0){
+solrClient.add(docList);
+solrClient.commit();
+docList.clear();
+ }
+ } catch (FileNotFoundExceptione){
+e.printStackTrace();
+logger.error("Error in Indexing file : "+file.getName(),e);
+ } catch (IOExceptione){
+e.printStackTrace();
+logger.error("Error in Indexing file : "+file.getName(),e);
+ } catch (SolrServerExceptione){
+e.printStackTrace();
+logger.error("Error in Indexing file : "+file.getName(),e);
+ }
+logger.info("Indexing Completed for file : "+file.getName());
+returntrue;
+ }
+
+publicvoidoptimizeIndex(){
+try{
+logger.info("Index Optimization Process Start");
+solrClient.commit();
+solrClient.optimize();
+logger.info("Index Optimization Process Completed");
+ } catch (SolrServerExceptione){
+e.printStackTrace();
+logger.error("Error in Index optimization process",e);
+ } catch (IOExceptione){
+e.printStackTrace();
+logger.error("Error in Index optimization process",e);
+ }
+
+ }
+
+}