diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c56fca7..0d9840eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Do not cancel context for async queries on success ([#859](https://github.com/src-d/go-mysql-server/pull/859)) +- sql: information schema column types should be lowercase ([#851](https://github.com/src-d/go-mysql-server/pull/851)) + +### Changed + +- Add progress for each partition in SHOW PROCESSLIST ([#855](https://github.com/src-d/go-mysql-server/pull/855)) +- Change BLAME to also take a file parameter. + +## [0.24.0-rc3] - 2019-10-23 + +### Fixed + +- Only optimize sorted DISTINCT if the first column on the order by is on the distinct schema. ([#976](https://github.com/src-d/gitbase/issues/976)) +- Avoid possible panics in LOC by using a safe cache accessor. +- sql: Add length to VARCHAR and CHAR MySQLTypeName strings in `SHOW CREATE TABLE` output. + +### Added + +- Added BLAME function. +- Better error messages for index failures. +- Implemented subquery expressions. +- Added support for 24-bit integers (MySQL's MEDIUMINT) + +### Changed + +- Use the minimum integer size as necessary when parsing literals. + +## [0.24.0-rc2] - 2019-10-02 + +### Fixed + +- plan: return types in lowercase in SHOW CREATE TABLE ([#827](https://github.com/src-d/go-mysql-server/pull/827)) +- analyzer: do not erase sort node when pushing it down ([#818](https://github.com/src-d/go-mysql-server/pull/818)) +- Fixed null errors during value comparisons ([#831](https://github.com/src-d/go-mysql-server/pull/831)) +- plan: fix race conditions in Exchange node +- Add CHAR and DATETIME types support ([#823](https://github.com/src-d/go-mysql-server/pull/823)) +- Also check sockets bind to tcp6 and fail on all closed sockets ([#824](https://github.com/src-d/go-mysql-server/pull/824)) + +### Changed + +- Added LIKE test with newlines ([#820](https://github.com/src-d/go-mysql-server/pull/820)) +- Convert LIKE patterns to specific Go regexes ([#817](https://github.com/src-d/go-mysql-server/pull/817)) + ## [0.24.0-rc1] - 2019-09-19 ### Added diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..f48cffb3a --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @src-d/data-processing diff --git a/Jenkinsfile b/Jenkinsfile index 065f9fbd8..332d5a5b8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,7 +7,7 @@ pipeline { nodeSelector 'srcd.host/type=jenkins-worker' containerTemplate { name 'regression-gitbase' - image 'srcd/regression-gitbase:v0.3.3' + image 'srcd/regression-gitbase:v0.3.4' ttyEnabled true command 'cat' } @@ -29,12 +29,6 @@ pipeline { sh '/bin/regression --complexity=2 --csv --prom local:HEAD' } } - stage('Run bblfsh mockup tests') { - when { branch 'master' } - steps { - sh '/bin/regression-bblfsh-mockups local:HEAD' - } - } stage('PR-run') { when { changeRequest target: 'master' } steps { @@ -66,5 +60,19 @@ pipeline { } } } + stage('Run bblfsh mockup tests') { + when { branch 'master' } + steps { + sh '/bin/regression-bblfsh-mockups local:HEAD' + } + } + } + post { + success { + slackSend (color: '#2eb886', message: "SUCCESS: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") + } + failure { + slackSend (color: '#b82e60', message: "FAILED: `${env.JOB_NAME}` <${env.BUILD_URL}|build #${env.BUILD_NUMBER}>") + } } } diff --git a/MAINTAINERS b/MAINTAINERS index dc6817245..3ae18cb82 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1 +1,2 @@ +Miguel Molina (@erizocosmico) Antonio Navarro Perez (@ajnavarro) diff --git a/docs/using-gitbase/examples.md b/docs/using-gitbase/examples.md index ef0007da4..5b54e65b6 100644 --- a/docs/using-gitbase/examples.md +++ b/docs/using-gitbase/examples.md @@ -78,8 +78,8 @@ GROUP BY committer_email, SELECT LANGUAGE(file_path, blob_content) as lang, SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Code')) as code, - SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Comments')) as comments, - SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Blanks')) as blanks, + SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Comment')) as comments, + SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Blank')) as blanks, COUNT(1) as files FROM refs NATURAL JOIN commit_files @@ -200,6 +200,41 @@ The output will be similar to this: +-----------------+--------------------------------------+---------------+------------------+--------------------+ ``` +## Get miscelaneous information about lines with a "// TODO" comment in HEAD + +```sql +SELECT repository_id, file_path, + JSON_UNQUOTE(JSON_EXTRACT(bl, "$.linenum")), + JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author")), + JSON_UNQUOTE(JSON_EXTRACT(bl, "$.text")) +FROM (SELECT repository_id, file_path, + EXPLODE(BLAME(repository_id, commit_hash, file_path)) AS bl + FROM ref_commits + NATURAL JOIN blobs + NATURAL JOIN commit_files + WHERE ref_name = 'HEAD' + AND NOT IS_BINARY(blob_content) + ) as p +WHERE JSON_EXTRACT(bl, "$.text") LIKE '%// TODO%'; +``` + +## Report of authors with more lines authored in HEAD + +```sql +SELECT + JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author")), + COUNT(JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author"))) + +FROM (SELECT EXPLODE(BLAME(repository_id, commit_hash, file_path)) AS bl + FROM ref_commits + NATURAL JOIN blobs + NATURAL JOIN commit_files + WHERE ref_name = 'HEAD' + AND NOT IS_BINARY(blob_content) + ) AS p +GROUP BY JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author")); +``` + # UAST UDFs Examples First of all, you should check out the [bblfsh documentation](https://docs.sourced.tech/babelfish) to get yourself familiar with UAST concepts. diff --git a/docs/using-gitbase/functions.md b/docs/using-gitbase/functions.md index c29fe7581..8da8c1533 100644 --- a/docs/using-gitbase/functions.md +++ b/docs/using-gitbase/functions.md @@ -6,19 +6,22 @@ To make some common tasks easier for the user, there are some functions to inter | Name | Description | |:-------------|:-------------------------------------------------------------------------------------------------------------------------------| -|`commit_stats(repository_id, [from_commit_hash], to_commit_hash) json`|returns the stats between two commits for a repository. If `from_commit_hash` is empty, it will compare the given `to_commit_hash` with its parent commit. Vendored files stats are not included in the result of this function. This function is more thoroughly explained later in this document.| +|`blame(repository, commit, file)`|Returns an array of lines changes and authorship for the specific file and commit. |`commit_file_stats(repository_id, [from_commit_hash], to_commit_hash) json array`|returns an array with the stats of each file in `to_commit_hash` since the given `from_commit_hash`. If `from_commit_hash` is not given, the parent commit will be used. Vendored files stats are not included in the result of this function. This function is more thoroughly explained later in this document.| -|`is_remote(reference_name)bool`| checks if the given reference name is from a remote one. | -|`is_tag(reference_name)bool`| checks if the given reference name is a tag. | -|`is_vendor(file_path)bool`| checks if the given file name is a vendored file. | +|`commit_stats(repository_id, [from_commit_hash], to_commit_hash) json`|returns the stats between two commits for a repository. If `from_commit_hash` is empty, it will compare the given `to_commit_hash` with its parent commit. Vendored files stats are not included in the result of this function. This function is more thoroughly explained later in this document.| +|`is_remote(reference_name)bool`| checks if the given reference name is from a remote one. | +|`is_tag(reference_name)bool`| checks if the given reference name is a tag. | +|`is_vendor(file_path)bool`| checks if the given file name is a vendored file. | |`language(path, [blob])text`| gets the language of a file given its path and the optional content of the file. | +|`loc(path, blob) json`| returns a JSON map, containing the lines of code of a file, separated in three categories: Code, Blank and Comment lines. | |`uast(blob, [lang, [xpath]]) blob`| returns a node array of UAST nodes in semantic mode. | +|`uast_children(blob) blob`| returns a flattened array of the children UAST nodes from each one of the UAST nodes in the given array. | +|`uast_extract(blob, key) text array`| extracts information identified by the given key from the uast nodes. | +|`uast_imports(blob) text array`| returns all imports given the specified UAST blob. | |`uast_mode(mode, blob, lang) blob`| returns a node array of UAST nodes specifying its language and mode (semantic, annotated or native). | |`uast_xpath(blob, xpath) blob`| performs an XPath query over the given UAST nodes. | -|`uast_extract(blob, key) text array`| extracts information identified by the given key from the uast nodes. | -|`uast_children(blob) blob`| returns a flattened array of the children UAST nodes from each one of the UAST nodes in the given array. | -|`loc(path, blob) json`| returns a JSON map, containing the lines of code of a file, separated in three categories: Code, Blank and Comment lines. | |`version() text`| returns the gitbase version in the following format `8.0.11-{GITBASE_VERSION}` for compatibility with MySQL versioning. | + ## Standard functions These are all functions that are available because they are implemented in `go-mysql-server`, used by gitbase. diff --git a/docs/using-gitbase/supported-syntax.md b/docs/using-gitbase/supported-syntax.md index a316dffe1..87841ac72 100644 --- a/docs/using-gitbase/supported-syntax.md +++ b/docs/using-gitbase/supported-syntax.md @@ -27,7 +27,6 @@ - ALIAS (AS) - CAST/CONVERT - CREATE TABLE -- DESCRIBE/DESC/EXPLAIN [table name] - DESCRIBE/DESC/EXPLAIN FORMAT=TREE [query] - DISTINCT - FILTER (WHERE) @@ -80,9 +79,6 @@ - div - % -## Subqueries -- supported only as tables, not as expressions. - ## Functions - ARRAY_LENGTH - CEIL @@ -133,3 +129,6 @@ - WEEKDAY - YEAR - YEARWEEK + +## Subqueries +Supported both as a table and as expressions but they can't access the parent query scope. diff --git a/go.mod b/go.mod index c52c87ff4..93ab93f08 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.12 require ( github.com/bblfsh/go-client/v4 v4.1.0 github.com/bblfsh/sdk/v3 v3.2.2 - github.com/gliderlabs/ssh v0.2.0 // indirect github.com/go-kit/kit v0.8.0 github.com/go-sql-driver/mysql v1.4.1 github.com/gorilla/handlers v1.4.0 // indirect @@ -16,22 +15,24 @@ require ( github.com/prometheus/client_golang v1.0.0 github.com/sirupsen/logrus v1.4.2 github.com/src-d/enry/v2 v2.0.0 - github.com/src-d/go-borges v0.1.3 + github.com/src-d/go-borges v0.1.4-0.20191017133700-c26ddd90fcbd github.com/src-d/go-git v4.7.0+incompatible github.com/src-d/go-git-fixtures v3.5.1-0.20190605154830-57f3972b0248+incompatible - github.com/src-d/go-mysql-server v0.4.1-0.20190821121850-0e0249cf7bc0 + github.com/src-d/go-mysql-server v0.6.1-0.20191028091010-5b6820662f02 github.com/stretchr/testify v1.3.0 github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.0.0+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect - golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b // indirect - golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect + golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect google.golang.org/grpc v1.20.1 gopkg.in/src-d/go-billy-siva.v4 v4.6.0 gopkg.in/src-d/go-billy.v4 v4.3.2 gopkg.in/src-d/go-errors.v1 v1.0.0 - gopkg.in/src-d/go-git.v4 v4.12.0 + gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 + gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v2 v2.2.2 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible ) diff --git a/go.sum b/go.sum index 958df669b..c1adf709b 100644 --- a/go.sum +++ b/go.sum @@ -68,10 +68,9 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/gliderlabs/ssh v0.1.3/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/gliderlabs/ssh v0.1.4/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/gliderlabs/ssh v0.2.0 h1:x0lYvhr3g30Vo8ISP+XgrP1KoC0/BiUUTY/HsosqSS4= -github.com/gliderlabs/ssh v0.2.0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -141,6 +140,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kevinburke/go-bindata v3.13.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= @@ -244,19 +245,20 @@ github.com/src-d/enry/v2 v2.0.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= -github.com/src-d/go-borges v0.1.3 h1:EgNqVy6Mw6wT3mx5AIUWjOAJJJQ6yuiDimi8JV/R2hE= -github.com/src-d/go-borges v0.1.3/go.mod h1:2lbHENELjpD+6bTOOnrnGEpVPMmO+wBGtYwzvQdr/Zc= +github.com/src-d/go-borges v0.1.4-0.20191017133700-c26ddd90fcbd h1:V44aPsmYy/G89yVNwz90Z4JFuZE3kDRb8n7qga6ZKEA= +github.com/src-d/go-borges v0.1.4-0.20191017133700-c26ddd90fcbd/go.mod h1:2lbHENELjpD+6bTOOnrnGEpVPMmO+wBGtYwzvQdr/Zc= github.com/src-d/go-git v4.7.0+incompatible h1:IYSSnbAHeKmsfbQFi9ozbid+KNh0bKjlorMfQehQbcE= github.com/src-d/go-git v4.7.0+incompatible/go.mod h1:1bQciz+hn0jzPQNsYj0hDFZHLJBdV7gXE2mWhC7EkFk= github.com/src-d/go-git-fixtures v3.5.1-0.20190605154830-57f3972b0248+incompatible h1:A5bKevhs9C//Nh8QV0J+1KphEaIa25cDe1DTs/yPxDI= github.com/src-d/go-git-fixtures v3.5.1-0.20190605154830-57f3972b0248+incompatible/go.mod h1:XcIQp7L+k0pgfTqfbaTKj3kxlBv8kYOKZ/tKNXbZFLg= -github.com/src-d/go-mysql-server v0.4.1-0.20190821121850-0e0249cf7bc0 h1:FaAetYRjr0fV+I44rqjgAoM6FZK2S1GmWuphk0CIpDU= -github.com/src-d/go-mysql-server v0.4.1-0.20190821121850-0e0249cf7bc0/go.mod h1:DdWE0ku/mNfuLsRJIrHeHpDtB7am+6oopxEsQKmVkx8= +github.com/src-d/go-mysql-server v0.6.1-0.20191028091010-5b6820662f02 h1:juAlqqpAYsS25PXJUaT0fV7SnutekViMcfz4lAPcSHU= +github.com/src-d/go-mysql-server v0.6.1-0.20191028091010-5b6820662f02/go.mod h1:DdWE0ku/mNfuLsRJIrHeHpDtB7am+6oopxEsQKmVkx8= github.com/src-d/go-oniguruma v1.0.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM= github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww= github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -288,10 +290,11 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -305,11 +308,12 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190520210107-018c4d40a106/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 h1:p9xBe/w/OzkeYVKm234g55gMdD1nSIooTir5kV11kfA= +golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -332,6 +336,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -340,6 +346,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -366,8 +373,6 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKW gopkg.in/src-d/go-billy-siva.v4 v4.6.0 h1:HO5m7lqYewIZ3Otay3IkQg3gFznW8Gy9HIbHWm1mYX0= gopkg.in/src-d/go-billy-siva.v4 v4.6.0/go.mod h1:EcgzPxovlWGD+lZFFriUleL3EVZ/SPs6CH2FOE/eooI= gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= -gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= -gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/src-d/go-errors.v1 v1.0.0 h1:cooGdZnCjYbeS1zb1s6pVAAimTdKceRrpn7aKOnNIfc= @@ -376,8 +381,8 @@ gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzW gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk= -gopkg.in/src-d/go-git.v4 v4.12.0 h1:CKgvBCJCcdfNnyXPYI4Cp8PaDDAmAPEN0CtfEdEAbd8= -gopkg.in/src-d/go-git.v4 v4.12.0/go.mod h1:zjlNnzc1Wjn43v3Mtii7RVxiReNP0fIu9npcXKzuNp4= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= +gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/src-d/go-log.v1 v1.0.2/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/src-d/go-siva.v1 v1.7.0 h1:igjgSEFweZ2kEfRlGEJH767o8GJRiPWp8JmHDCe0Vdk= gopkg.in/src-d/go-siva.v1 v1.7.0/go.mod h1:ChxMHSRkICHZ9IbTlG3ihkuG7gc2RZPsIYh7OaXYvic= diff --git a/integration_test.go b/integration_test.go index 6cb93e91b..eb91f5085 100644 --- a/integration_test.go +++ b/integration_test.go @@ -509,6 +509,18 @@ func TestIntegration(t *testing.T) { {"vendor/foo.go"}, }, }, + { + ` + SELECT repository_id, JSON_EXTRACT(bl, "$.author"), + COUNT(bl) + FROM ( + SELECT repository_id, EXPLODE(BLAME(repository_id, commit_hash, '.gitignore')) as bl + FROM commits + WHERE commit_hash = '918c48b83bd081e863dbe1b80f8998f058cd8294' + ) as p + `, + []sql.Row{{"worktree", "mcuadros@gmail.com", int64(12)}}, + }, } var pid uint64 diff --git a/internal/function/blame.go b/internal/function/blame.go new file mode 100644 index 000000000..0bf9c0a2f --- /dev/null +++ b/internal/function/blame.go @@ -0,0 +1,178 @@ +package function + +import ( + "fmt" + "io" + + "github.com/src-d/gitbase" + "github.com/src-d/go-mysql-server/sql" + "gopkg.in/src-d/go-git.v4" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" +) + +type BlameGenerator struct { + ctx *sql.Context + commit *object.Commit + file string + curLine int + lines []*git.Line +} + +func NewBlameGenerator(ctx *sql.Context, c *object.Commit, f string) (*BlameGenerator, error) { + result, err := git.Blame(c, f) + if err != nil { + return nil, err + } + return &BlameGenerator{ + ctx: ctx, + commit: c, + file: f, + curLine: 0, + lines: result.Lines, + }, nil +} + +func (g *BlameGenerator) Next() (interface{}, error) { + select { + case <-g.ctx.Done(): + return nil, io.EOF + default: + } + + if len(g.lines) == 0 || g.curLine >= len(g.lines) { + return nil, io.EOF + } + + l := g.lines[g.curLine] + b := BlameLine{ + LineNum: g.curLine, + Author: l.Author, + Text: l.Text, + } + g.curLine++ + return b, nil +} + +func (g *BlameGenerator) Close() error { + return nil +} + +var _ sql.Generator = (*BlameGenerator)(nil) + +type ( + // Blame implements git-blame function as UDF + Blame struct { + repo sql.Expression + commit sql.Expression + file sql.Expression + } + + // BlameLine represents each line of git blame's output + BlameLine struct { + LineNum int `json:"linenum"` + Author string `json:"author"` + Text string `json:"text"` + } +) + +// NewBlame constructor +func NewBlame(repo, commit, file sql.Expression) sql.Expression { + return &Blame{repo, commit, file} +} + +func (b *Blame) String() string { + return fmt.Sprintf("blame(%s, %s)", b.repo, b.commit) +} + +// Type implements the sql.Expression interface +func (*Blame) Type() sql.Type { + return sql.Array(sql.JSON) +} + +func (b *Blame) WithChildren(children ...sql.Expression) (sql.Expression, error) { + if len(children) != 3 { + return nil, sql.ErrInvalidChildrenNumber.New(b, len(children), 2) + } + + return NewBlame(children[0], children[1], children[2]), nil +} + +// Children implements the Expression interface. +func (b *Blame) Children() []sql.Expression { + return []sql.Expression{b.repo, b.commit, b.file} +} + +// IsNullable implements the Expression interface. +func (b *Blame) IsNullable() bool { + return b.repo.IsNullable() || (b.commit.IsNullable()) || (b.file.IsNullable()) +} + +// Resolved implements the Expression interface. +func (b *Blame) Resolved() bool { + return b.repo.Resolved() && b.commit.Resolved() && b.file.Resolved() +} + +// Eval implements the sql.Expression interface. +func (b *Blame) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + span, ctx := ctx.Span("gitbase.Blame") + defer span.Finish() + + repo, err := b.resolveRepo(ctx, row) + if err != nil { + ctx.Warn(0, err.Error()) + return nil, nil + } + + commit, err := b.resolveCommit(ctx, repo, row) + if err != nil { + ctx.Warn(0, err.Error()) + return nil, nil + } + + file, err := exprToString(ctx, b.file, row) + if err != nil { + ctx.Warn(0, err.Error()) + return nil, nil + } + + bg, err := NewBlameGenerator(ctx, commit, file) + if err != nil { + ctx.Warn(0, err.Error()) + return nil, nil + } + + return bg, nil +} + +func (b *Blame) resolveCommit(ctx *sql.Context, repo *gitbase.Repository, row sql.Row) (*object.Commit, error) { + str, err := exprToString(ctx, b.commit, row) + if err != nil { + return nil, err + } + + commitHash, err := repo.ResolveRevision(plumbing.Revision(str)) + if err != nil { + h := plumbing.NewHash(str) + commitHash = &h + } + to, err := repo.CommitObject(*commitHash) + if err != nil { + return nil, err + } + + return to, nil +} + +func (b *Blame) resolveRepo(ctx *sql.Context, r sql.Row) (*gitbase.Repository, error) { + repoID, err := exprToString(ctx, b.repo, r) + if err != nil { + return nil, err + } + s, ok := ctx.Session.(*gitbase.Session) + if !ok { + return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session) + } + return s.Pool.GetRepo(repoID) +} diff --git a/internal/function/blame_test.go b/internal/function/blame_test.go new file mode 100644 index 000000000..aacdee15b --- /dev/null +++ b/internal/function/blame_test.go @@ -0,0 +1,132 @@ +package function + +import ( + "context" + "testing" + + "github.com/src-d/gitbase" + "github.com/src-d/go-mysql-server/sql" + "github.com/src-d/go-mysql-server/sql/expression" + "github.com/stretchr/testify/require" + fixtures "gopkg.in/src-d/go-git-fixtures.v3" +) + +func TestBlameEval(t *testing.T) { + require.NoError(t, fixtures.Init()) + + defer func() { + require.NoError(t, fixtures.Clean()) + }() + + pool, cleanup := setupPool(t) + defer cleanup() + + session := gitbase.NewSession(pool) + ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) + + testCases := []struct { + name string + repo sql.Expression + commit sql.Expression + file sql.Expression + row sql.Row + expected BlameLine + expectedNil bool + testedLine int + lineCount int + }{ + { + name: "init commit", + repo: expression.NewGetField(0, sql.Text, "repository_id", false), + commit: expression.NewGetField(1, sql.Text, "commit_hash", false), + file: expression.NewGetField(2, sql.Text, "file", false), + row: sql.NewRow("worktree", "b029517f6300c2da0f4b651b8642506cd6aaf45d", ".gitignore"), + testedLine: 0, + lineCount: 12, + expected: BlameLine{ + 0, + "mcuadros@gmail.com", + "*.class", + }, + expectedNil: false, + }, + { + name: "changelog", + repo: expression.NewGetField(0, sql.Text, "repository_id", false), + commit: expression.NewGetField(1, sql.Text, "commit_hash", false), + file: expression.NewGetField(2, sql.Text, "file", false), + row: sql.NewRow("worktree", "b8e471f58bcbca63b07bda20e428190409c2db47", "CHANGELOG"), + testedLine: 0, + lineCount: 1, + expected: BlameLine{ + 0, + "daniel@lordran.local", + "Initial changelog", + }, + expectedNil: false, + }, + { + name: "no repo", + repo: expression.NewGetField(0, sql.Text, "repository_id", false), + commit: expression.NewGetField(1, sql.Text, "commit_hash", false), + file: expression.NewGetField(2, sql.Text, "file", false), + row: sql.NewRow("foo", "bar", "baz"), + testedLine: 0, + lineCount: 1, + expected: BlameLine{}, + expectedNil: true, + }, + { + name: "no commit", + repo: expression.NewGetField(0, sql.Text, "repository_id", false), + commit: expression.NewGetField(1, sql.Text, "commit_hash", false), + file: expression.NewGetField(2, sql.Text, "file", false), + row: sql.NewRow("worktree", "foo", "bar"), + testedLine: 0, + lineCount: 1, + expected: BlameLine{}, + expectedNil: true, + }, + { + name: "no file", + repo: expression.NewGetField(0, sql.Text, "repository_id", false), + commit: expression.NewGetField(1, sql.Text, "commit_hash", false), + file: expression.NewGetField(2, sql.Text, "file", false), + row: sql.NewRow("worktree", "b8e471f58bcbca63b07bda20e428190409c2db47", "foo"), + testedLine: 0, + lineCount: 1, + expected: BlameLine{}, + expectedNil: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + blame := NewBlame(tc.repo, tc.commit, tc.file) + blameGen, err := blame.Eval(ctx, tc.row) + require.NoError(t, err) + + if tc.expectedNil { + require.Nil(t, blameGen) + return + } else { + require.NotNil(t, blameGen) + } + + bg := blameGen.(*BlameGenerator) + defer bg.Close() + + lineCount := 0 + for i, err := bg.Next(); err == nil; i, err = bg.Next() { + i := i.(BlameLine) + if lineCount != tc.testedLine { + lineCount++ + continue + } + lineCount++ + require.EqualValues(t, tc.expected, i) + } + require.Equal(t, tc.lineCount, lineCount) + }) + } +} diff --git a/internal/function/loc.go b/internal/function/loc.go index 35861fc9d..0476f30c8 100644 --- a/internal/function/loc.go +++ b/internal/function/loc.go @@ -75,7 +75,7 @@ func (f *LOC) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { return nil, err } - lang, err := f.getLanguage(path, blob) + lang, err := getLanguage(ctx, path, blob) if err != nil { return nil, err } @@ -142,9 +142,10 @@ func (f *LOC) getInputValues(ctx *sql.Context, row sql.Row) (string, []byte, err return path, blob, nil } -func (f *LOC) getLanguage(path string, blob []byte) (string, error) { +func getLanguage(ctx *sql.Context, path string, blob []byte) (string, error) { hash := languageHash(path, blob) + languageCache := getLanguageCache(ctx) value, err := languageCache.Get(hash) if err == nil { return value.(string), nil diff --git a/internal/function/registry.go b/internal/function/registry.go index e4a25db13..a059e674d 100644 --- a/internal/function/registry.go +++ b/internal/function/registry.go @@ -17,4 +17,5 @@ var Functions = []sql.Function{ sql.Function1{Name: "uast_children", Fn: NewUASTChildren}, sql.Function1{Name: "uast_imports", Fn: NewUASTImports}, sql.Function1{Name: "is_vendor", Fn: NewIsVendor}, + sql.Function3{Name: "blame", Fn: NewBlame}, }