Merge pull request harelba#143 from harelba/v1.6.0-release-test

V1.6.2 stuff

Author: harelba

.gitignore

@@ -8,3 +8,5 @@ rpm_build_area
 setup.exe
 win_output
 win_build
+packages
+.idea/

bin/q

@@ -27,7 +27,7 @@
 #
 # Run with --help for command line details
 #
-q_version="1.6.0notreleasedyet"
+q_version="1.6.0"
 
 __all__= [ 'QTextAsData' ]
 
@@ -50,8 +50,6 @@ import uuid
 importcStringIO
 importmath
 
-csv.field_size_limit(sys.maxsize)
-
 DEBUG=False
 
 defget_stdout_encoding(encoding_override=None):
@@ -210,6 +208,14 @@ class CouldNotConvertStringToNumericValueException(Exception):
 def__str(self):
 returnrepr(self.msg)
 
+classColumnMaxLengthLimitExceededException(Exception):
+
+def__init__(self, msg):
+self.msg=msg
+
+def__str(self):
+returnrepr(self.msg)
+
 classCouldNotParseInputException(Exception):
 
 def__init__(self, msg):
@@ -636,6 +642,11 @@ def encoded_csv_reader(encoding, f, dialect, **kwargs):
 raiseCouldNotConvertStringToNumericValueException(e.message)
 else:
 raiseCouldNotParseInputException(str(e))
+exceptException,e:
+ifstr(e).startswith("field larger than field limit"):
+raiseColumnMaxLengthLimitExceededException(str(e))
+else:
+raise
 
 defnormalized_filename(filename):
 iffilename=='-':
@@ -671,9 +682,13 @@ class MaterializedFileState(object):
 exceptException,e:
 raiseException('Tried to skip BOM for "utf-8-sig" encoding and failed. Error message is '+str(e))
 csv_reader=encoded_csv_reader(self.encoding, self.f, dialect=self.dialect)
-forcol_valsincsv_reader:
-self.lines_read+=1
-yieldcol_vals
+try:
+forcol_valsincsv_reader:
+self.lines_read+=1
+yieldcol_vals
+exceptColumnMaxLengthLimitExceededException,e:
+msg="Column length is larger than the maximum. Offending file is '%s' - Line is %s, counting from 1 (encoding %s). The line number is the raw line number of the file, ignoring whether there's a header or not"% (self.filename,self.lines_read+1,self.encoding)
+raiseColumnMaxLengthLimitExceededException(msg)
 
 defclose(self):
 ifself.f!=sys.stdin:
@@ -1079,7 +1094,8 @@ class QInputParams(object):
 expected_column_count=None,keep_leading_whitespace_in_values=False,
 disable_double_double_quoting=False,disable_escaped_double_quoting=False,
 disable_column_type_detection=False,
-input_quoting_mode='minimal',stdin_file=None,stdin_filename='-'):
+input_quoting_mode='minimal',stdin_file=None,stdin_filename='-',
+max_column_length_limit=131072):
 self.skip_header=skip_header
 self.delimiter=delimiter
 self.input_encoding=input_encoding
@@ -1091,6 +1107,7 @@ class QInputParams(object):
 self.disable_escaped_double_quoting=disable_escaped_double_quoting
 self.input_quoting_mode=input_quoting_mode
 self.disable_column_type_detection=disable_column_type_detection
+self.max_column_length_limit=max_column_length_limit
 
 defmerged_with(self,input_params):
 params=QInputParams(**self.__dict__)
@@ -1113,7 +1130,6 @@ class QTextAsData(object):
 # Create DB object
 self.db=Sqlite3DB()
 
-
 input_quoting_modes={'minimal' : csv.QUOTE_MINIMAL,
 'all' : csv.QUOTE_ALL,
 # nonnumeric is not supported for input quoting modes, since we determine the data types 
@@ -1149,6 +1165,8 @@ class QTextAsData(object):
 dialect_id=self.get_dialect_id(filename)
 csv.register_dialect(dialect_id, **q_dialect)
 
+csv.field_size_limit(input_params.max_column_length_limit)
+
 # Create a line splitter
 line_splitter=LineSplitter(input_params.delimiter, input_params.expected_column_count)
 
@@ -1261,6 +1279,8 @@ class QTextAsData(object):
 error=QError(e,"Could not convert string to a numeric value. Did you use `-w nonnumeric` with unquoted string values? Error: %s"%e.msg,58)
 exceptCouldNotParseInputException,e:
 error=QError(e,"Could not parse the input. Please make sure to set the proper -w input-wrapping parameter for your input, and that you use the proper input encoding (-e). Error: %s"%e.msg,59)
+exceptColumnMaxLengthLimitExceededException,e:
+error=QError(e,e.msg,31)
 exceptKeyboardInterrupt,e:
 warnings.append(QWarning(e,"Interrupted"))
 exceptException, e:
@@ -1530,6 +1550,8 @@ def run_standalone():
 #-----------------------------------------------
 parser.add_option("-v", "--version", dest="version", default=False, action="store_true",
 help="Print version")
+parser.add_option("-V", "--verbose", dest="verbose", default=False, action="store_true",
+help="Print debug info in case of problems")
 #-----------------------------------------------
 input_data_option_group=OptionGroup(parser,"Input Data Options")
 input_data_option_group.add_option("-H", "--skip-header", dest="skip_header", default=default_skip_header, action="store_true",
@@ -1558,6 +1580,8 @@ def run_standalone():
 help="Don't detect column types - All columns will be treated as text columns")
 input_data_option_group.add_option("-w","--input-quoting-mode",dest="input_quoting_mode",default="minimal",
 help="Input quoting mode. Possible values are all, minimal and none. Note the slightly misleading parameter name, and see the matching -W parameter for output quoting.")
+input_data_option_group.add_option("-M","--max-column-length-limit",dest="max_column_length_limit",default=131072,
+help="Sets the maximum column length.")
 parser.add_option_group(input_data_option_group)
 #-----------------------------------------------
 output_data_option_group=OptionGroup(parser,"Output Options") 
@@ -1681,6 +1705,14 @@ def run_standalone():
 # (since no input delimiter means any whitespace)
 options.output_delimiter=" "
 
+try:
+max_column_length_limit=int(options.max_column_length_limit)
+ifmax_column_length_limit<1:
+raiseException()
+except:
+print>>sys.stderr, "Max column length limit must be a positive integer (%s)"%max_column_length_limit
+sys.exit(31)
+
 default_input_params=QInputParams(skip_header=options.skip_header,
 delimiter=options.delimiter,
 input_encoding=options.encoding,
@@ -1691,7 +1723,8 @@ def run_standalone():
 disable_double_double_quoting=options.disable_double_double_quoting,
 disable_escaped_double_quoting=options.disable_escaped_double_quoting,
 input_quoting_mode=options.input_quoting_mode,
-disable_column_type_detection=options.disable_column_type_detection)
+disable_column_type_detection=options.disable_column_type_detection,
+max_column_length_limit=max_column_length_limit)
 q_engine=QTextAsData(default_input_params=default_input_params)
 
 output_params=QOutputParams(
@@ -1700,7 +1733,7 @@ def run_standalone():
 output_quoting_mode=options.output_quoting_mode,
 formatting=options.formatting,
 output_header=options.output_header)
-q_output_printer=QOutputPrinter(output_params,show_tracebacks=DEBUG)
+q_output_printer=QOutputPrinter(output_params,show_tracebacks=options.verbose)
 
 forquery_strinquery_strs:
 ifoptions.analyze_only:

build-deb-builder-container

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [ $#-ne 1 ];
+then
+echo"Usage: $(basename $0) <version-tag>"
+exit 1
+fi
+VERSION_TAG="$1"
+
+docker build -f dist/deb-builder-Dockerfile -t q-text-as-data-deb-builder:${VERSION_TAG}.

build-rpm-builder-container

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [ $#-ne 1 ];
+then
+echo"Usage: $(basename $0) <version-tag>"
+exit 1
+fi
+VERSION_TAG="$1"
+
+docker build -f dist/rpm-builder-Dockerfile -t q-text-as-data-rpm-builder:${VERSION_TAG}.

dist/create-rpm

@@ -5,9 +5,9 @@
 #
 #
 
-if [ $#-ne2 ];
+if [ $#-ne1 ];
 then
-echo'create-rpm <commit-hash> <version>'
+echo'create-rpm <version>'
 exit 1
 fi
 
@@ -26,9 +26,7 @@ mkdir -p ${rpm_build_area}/{SOURCES,SPECS,BUILD,RPMS,SRPMS,BUILDROOT}
 
 echo RPM build area is in${rpm_build_area}
 
-COMMIT_HASH=$1
-SHORT_HASH=${COMMIT_HASH:0:7}
-VERSION=$2
+VERSION=$1
 REAL_PACKAGE_NAME=q
 RPM_PACKAGE_NAME=q-text-as-data
 
@@ -40,11 +38,15 @@ then
 exit 1
 fi
 
-rm -vf ${rpm_build_area}/SOURCES/q-${COMMIT_HASH}.tar.gz
-
-curl -o ${rpm_build_area}/SOURCES/q-${COMMIT_HASH}.tar.gz -L -R "https://github.com/harelba/q/tarball/${COMMIT_HASH}"
+curl -o ${rpm_build_area}/SOURCES/q.tar.gz -L -R "https://github.com/harelba/q/tarball/${VERSION}"
+mkdir -p ${rpm_build_area}/SOURCES
+pushd${rpm_build_area}/SOURCES >/dev/null
+tar xvzf ./q.tar.gz --strip-components=1
+rm -vf ./q.tar.gz
+popd>/dev/null
+find ${rpm_build_area}/ -ls
 
-cat ${RPM_PACKAGE_NAME}.spec.template | sed "s/VERSION_PLACEHOLDER/$VERSION/g"| sed "s/COMMIT_HASH_PLACEHOLDER/${COMMIT_HASH}/g"| sed "s/SHORT_HASH_PLACEHOLDER/${SHORT_HASH}/g">${rpm_build_area}/SPECS/${RPM_PACKAGE_NAME}.spec
+cat ${RPM_PACKAGE_NAME}.spec.template | sed "s/VERSION_PLACEHOLDER/$VERSION/g">${rpm_build_area}/SPECS/${RPM_PACKAGE_NAME}.spec
 
 rpmbuild -v --define "_topdir ${rpm_build_area}" -ba ${rpm_build_area}/SPECS/${RPM_PACKAGE_NAME}.spec
 

dist/deb-builder-Dockerfile

@@ -0,0 +1,8 @@
+
+FROM ubuntu:12.04
+
+RUN apt-get update && apt-get install -y alien
+
+ENTRYPOINT "/bin/bash"
+
+

dist/q-text-as-data.spec.template

@@ -10,18 +10,19 @@ Summary: q - Text as Data
 Group: Applications/Text
 License: GPLv3
 URL: https://github.com/harelba/q
-Source: q-COMMIT_HASH_PLACEHOLDER.tar.gz
 BuildArch: noarch
 
 %description
 q allows to perform SQL-like statements on tabular text data.
 
 
 %prep
-%setup -qn harelba-q-SHORT_HASH_PLACEHOLDER
+cd %{_topdir}/BUILD
+cp -vrf %{_topdir}/SOURCES/* %{_topdir}/BUILD/
+chmod -Rf a+rX,u+w,g-w,o-w %{_topdir}/BUILD/
 
 %build
-ls -ltr
+cd %{_topdir}/BUILD
 ronn doc/USAGE.markdown
 
 %install
@@ -43,6 +44,9 @@ gzip ${RPM_BUILD_ROOT}%{_mandir}/man1/q.1
 %doc %_mandir/man1/q.1.gz
 
 %changelog
+*Wed Apr 05 2017 Harel Ben-Attia <[email protected]> 1.6.0-1
+- Moved RPM building to be dockerized
+- Removed the need for providing commit hashes
 *Fri Dec 12 2014 Harel Ben-Attia <[email protected]> 1.5.0-1
 - Moved stuff from create-rpm script into the rpm spec itself
 *Sat Jun 14 2014 Harel Ben-Attia <[email protected]> 1.4.0-1

dist/rpm-builder-Dockerfile

@@ -0,0 +1,12 @@
+
+FROM centos:centos6
+
+RUN yum install -y which curl gcc make rpm rpm-build
+
+RUN curl -sSL https://get.rvm.io | bash
+
+RUN /bin/bash -l -c "rvm install 2.4.1" && /bin/bash -l -c "gem install ronn"
+
+ENTRYPOINT "/bin/bash"
+
+

package-release

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e 
+
+base_folder=$(dirname $0)
+pushd${base_folder}>/dev/null
+
+if [ $#-ne 1 ];
+then
+echo"Usage: $(dirname $0) <git-tag>"
+echo
+echo"Note that the git tag must be pushed to github before doing this."
+exit 1
+fi
+TAG="$1"
+
+d=`pwd`
+cid1=`docker run -i -d -v ${d}:/q q-text-as-data-rpm-builder:0.1`
+cid2=`docker run -i -d -v ${d}:/q q-text-as-data-deb-builder:0.1`
+
+functionkill_container{
+ tmp=`docker kill${cid1}${cid2}`
+}
+trap kill_container EXIT
+
+rm -rvf ${base_folder}/packages
+mkdir -p ${base_folder}/packages
+
+sleep 1
+docker exec -it ${cid1} /bin/bash -i -c "/q/dist/create-rpm ${TAG}"
+
+docker cp ${cid1}:/q/dist/rpm_build_area/RPMS/noarch/q-text-as-data-${TAG}-1.el6.noarch.rpm ${base_folder}/packages/q-text-as-data-${TAG}-1.noarch.rpm
+
+docker exec -it ${cid2} /bin/bash -i -c "cd /q/packages && alien ./q-text-as-data-${TAG}-1.noarch.rpm"
+