diff --git a/.gitignore b/.gitignore index 16182c5d0..05dfac0b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ -/_site \ No newline at end of file +/_site +/.jekyll-metadata +/downloads/cheatsheets/*.aux +/downloads/cheatsheets/*.log +/.bundle +/vendor \ No newline at end of file diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 000000000..944880fa1 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.2.0 diff --git a/CNAME b/CNAME index 615972c11..e9fe306fb 100644 --- a/CNAME +++ b/CNAME @@ -1 +1 @@ -elixir-lang.org +elixir-lang.org \ No newline at end of file diff --git a/Gemfile b/Gemfile index 97355ea72..f24d05f4a 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,8 @@ source 'https://rubygems.org' gem 'github-pages' +gem 'json', '>= 2.0.0' +gem 'webrick', '>= 1.8' +gem 'csv', '~> 3.3' +gem 'base64', '~> 0.2.0' +gem 'bigdecimal', '~> 3.1' diff --git a/Gemfile.lock b/Gemfile.lock index e4471ec3e..f77a0774e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,130 +1,270 @@ GEM remote: https://rubygems.org/ specs: - activesupport (4.2.7) - i18n (~> 0.7) - json (~> 1.7, >= 1.7.7) - minitest (~> 5.1) - thread_safe (~> 0.3, >= 0.3.4) - tzinfo (~> 1.1) - addressable (2.4.0) + activesupport (7.0.7.2) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) + base64 (0.2.0) + bigdecimal (3.1.9) coffee-script (2.4.1) coffee-script-source execjs - coffee-script-source (1.10.0) - colorator (0.1) - ethon (0.9.0) - ffi (>= 1.3.0) - execjs (2.7.0) - faraday (0.9.2) - multipart-post (>= 1.2, < 3) - ffi (1.9.14) - gemoji (2.1.0) - github-pages (89) - activesupport (= 4.2.7) - github-pages-health-check (= 1.1.0) - jekyll (= 3.1.6) - jekyll-coffeescript (= 1.0.1) - jekyll-feed (= 0.5.1) - jekyll-gist (= 1.4.0) - jekyll-github-metadata (= 2.0.2) - jekyll-mentions (= 1.1.3) + coffee-script-source (1.11.1) + colorator (1.1.0) + commonmarker (0.23.10) + concurrent-ruby (1.2.2) + csv (3.3.3) + dnsruby (1.61.9) + simpleidn (~> 0.1) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + ethon (0.16.0) + ffi (>= 1.15.0) + eventmachine (1.2.7) + execjs (2.8.1) + faraday (2.7.4) + faraday-net_http (>= 2.0, < 3.1) + ruby2_keywords (>= 0.0.4) + faraday-net_http (3.0.2) + ffi (1.15.5) + forwardable-extended (2.6.0) + gemoji (3.0.1) + github-pages (228) + github-pages-health-check (= 1.17.9) + jekyll (= 3.9.3) + jekyll-avatar (= 0.7.0) + jekyll-coffeescript (= 1.1.1) + jekyll-commonmark-ghpages (= 0.4.0) + jekyll-default-layout (= 0.1.4) + jekyll-feed (= 0.15.1) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.13.0) + jekyll-include-cache (= 0.2.1) + jekyll-mentions (= 1.6.0) + jekyll-optional-front-matter (= 0.3.2) jekyll-paginate (= 1.1.0) - jekyll-redirect-from (= 0.11.0) - jekyll-sass-converter (= 1.3.0) - jekyll-seo-tag (= 2.0.0) - jekyll-sitemap (= 0.10.0) - jemoji (= 0.7.0) - kramdown (= 1.11.1) - liquid (= 3.0.6) - listen (= 3.0.6) + jekyll-readme-index (= 0.3.0) + jekyll-redirect-from (= 0.16.0) + jekyll-relative-links (= 0.6.1) + jekyll-remote-theme (= 0.4.3) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.8.0) + jekyll-sitemap (= 1.4.0) + jekyll-swiss (= 1.0.0) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) + jekyll-titles-from-headings (= 0.5.3) + jemoji (= 0.12.0) + kramdown (= 2.3.2) + kramdown-parser-gfm (= 1.1.0) + liquid (= 4.0.4) mercenary (~> 0.3) - rouge (= 1.11.1) + minima (= 2.5.1) + nokogiri (>= 1.13.6, < 2.0) + rouge (= 3.26.0) terminal-table (~> 1.4) - github-pages-health-check (1.1.0) + github-pages-health-check (1.17.9) addressable (~> 2.3) - net-dns (~> 0.8) + dnsruby (~> 1.60) octokit (~> 4.0) - public_suffix (~> 1.4) - typhoeus (~> 0.7) - html-pipeline (2.4.2) + public_suffix (>= 3.0, < 5.0) + typhoeus (~> 1.3) + html-pipeline (2.14.3) activesupport (>= 2) nokogiri (>= 1.4) - i18n (0.7.0) - jekyll (3.1.6) - colorator (~> 0.1) + http_parser.rb (0.8.0) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + jekyll (3.9.3) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (>= 0.7, < 2) jekyll-sass-converter (~> 1.0) - jekyll-watch (~> 1.1) - kramdown (~> 1.3) - liquid (~> 3.0) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) + liquid (~> 4.0) mercenary (~> 0.3.3) - rouge (~> 1.7) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) safe_yaml (~> 1.0) - jekyll-coffeescript (1.0.1) + jekyll-avatar (0.7.0) + jekyll (>= 3.0, < 5.0) + jekyll-coffeescript (1.1.1) coffee-script (~> 2.2) - jekyll-feed (0.5.1) - jekyll-gist (1.4.0) + coffee-script-source (~> 1.11.1) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.4.0) + commonmarker (~> 0.23.7) + jekyll (~> 3.9.0) + jekyll-commonmark (~> 1.4.0) + rouge (>= 2.0, < 5.0) + jekyll-default-layout (0.1.4) + jekyll (~> 3.0) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-gist (1.5.0) octokit (~> 4.2) - jekyll-github-metadata (2.0.2) - jekyll (~> 3.1) - octokit (~> 4.0) - jekyll-mentions (1.1.3) + jekyll-github-metadata (2.13.0) + jekyll (>= 3.4, < 5.0) + octokit (~> 4.0, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-mentions (1.6.0) html-pipeline (~> 2.3) - jekyll (~> 3.0) + jekyll (>= 3.7, < 5.0) + jekyll-optional-front-matter (0.3.2) + jekyll (>= 3.0, < 5.0) jekyll-paginate (1.1.0) - jekyll-redirect-from (0.11.0) - jekyll (>= 2.0) - jekyll-sass-converter (1.3.0) - sass (~> 3.2) - jekyll-seo-tag (2.0.0) - jekyll (~> 3.1) - jekyll-sitemap (0.10.0) - jekyll-watch (1.5.0) - listen (~> 3.0, < 3.1) - jemoji (0.7.0) - activesupport (~> 4.0) - gemoji (~> 2.0) + jekyll-readme-index (0.3.0) + jekyll (>= 3.0, < 5.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-remote-theme (0.4.3) + addressable (~> 2.0) + jekyll (>= 3.5, < 5.0) + jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) + rubyzip (>= 1.3.0, < 3.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-swiss (1.0.0) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.6.0) + jekyll (> 3.5, < 5.0) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.3) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.12.0) + gemoji (~> 3.0) html-pipeline (~> 2.2) - jekyll (>= 3.0) - json (1.8.3) - kramdown (1.11.1) - liquid (3.0.6) - listen (3.0.6) - rb-fsevent (>= 0.9.3) - rb-inotify (>= 0.9.7) + jekyll (>= 3.0, < 5.0) + json (2.6.3) + kramdown (2.3.2) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.3.6) - mini_portile2 (2.1.0) - minitest (5.9.0) - multipart-post (2.0.0) - net-dns (0.8.0) - nokogiri (1.6.8) - mini_portile2 (~> 2.1.0) - pkg-config (~> 1.1.7) - octokit (4.3.0) - sawyer (~> 0.7.0, >= 0.5.3) - pkg-config (1.1.7) - public_suffix (1.5.3) - rb-fsevent (0.9.7) - rb-inotify (0.9.7) - ffi (>= 0.5.0) - rouge (1.11.1) - safe_yaml (1.0.4) - sass (3.4.22) - sawyer (0.7.0) - addressable (>= 2.3.5, < 2.5) - faraday (~> 0.8, < 0.10) - terminal-table (1.6.0) - thread_safe (0.3.5) - typhoeus (0.8.0) - ethon (>= 0.8.0) - tzinfo (1.2.2) - thread_safe (~> 0.1) + mini_portile2 (2.8.9) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.19.0) + nokogiri (1.18.9) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.7) + racc (1.8.1) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.4.2) + rouge (3.26.0) + ruby2_keywords (0.0.5) + rubyzip (2.3.2) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) + simpleidn (0.2.1) + unf (~> 0.1.4) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) + unicode-display_width (1.8.0) + webrick (1.8.2) PLATFORMS ruby DEPENDENCIES + base64 (~> 0.2.0) + bigdecimal (~> 3.1) + csv (~> 3.3) github-pages + json (>= 2.0.0) + webrick (>= 1.8) BUNDLED WITH - 1.11.2 + 2.5.23 diff --git a/README.md b/README.md index a003e2df7..a091828d8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -This projects holds the contents for Elixir website hosted at elixir-lang.org. +This projects holds the contents for the Elixir website hosted at [elixir-lang.org](https://elixir-lang.org). -It is automatically transformed by [Jekyll](https://github.com/mojombo/jekyll) into a static site. +It is automatically transformed by [Jekyll](https://github.com/jekyll/jekyll) into a static site. ## Contributing @@ -47,7 +47,7 @@ $ bundle exec jekyll serve ``` The generated site will be available at [http://localhost:4000](http://localhost:4000). You can stop the -server with Ctrl-C. +server with Ctrl+C. #### 5. Make your changes and push them @@ -61,12 +61,10 @@ guide](https://github.com/elixir-lang/elixir/#contributing). ## License -* "Elixir" and the Elixir logo are copyrighted to [Plataformatec](http://plataformatec.com.br/). You may not reuse anything therein without permission. +* "Elixir" and the Elixir logo are registered trademarks of the Elixir team. See [our trademark policy](https://elixir-lang.org/trademarks). * The HTML and CSS are copyrighted to [AlienWp](http://alienwp.com/) under [GPL license, version 2](http://www.gnu.org/licenses/old-licenses/gpl-2.0.html). * The Social Icons are copyrighted to [Xeloader](http://xeloader.deviantart.com/art/Socialis-2-Freebie-213292616). -* The written textual contents available in the guides and blog are licensed under Apache 2.0. - -* The available docs are licensed under the same license as their projects. +* The written textual contents available in the guides and blog are licensed under [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). diff --git a/_config.yml b/_config.yml index 2e55119e6..635db0b94 100644 --- a/_config.yml +++ b/_config.yml @@ -3,7 +3,18 @@ permalink: none kramdown: input: GFM hard_wrap: false -gems: +repository: elixir-lang/elixir-lang.github.com +url: https://elixir-lang.org +exclude: + - _build/ + - vendor/ + - Gemfile + - Gemfile.lock + - CNAME + - .gitignore + - README.md + - js/toc/README.md +plugins: - jemoji - jekyll-sitemap - jekyll-redirect-from @@ -15,3 +26,4 @@ defaults: values: layout: post permalink: /blog/:year/:month/:day/:title/ + image: /images/social/elixir-og-card.jpg diff --git a/_data/elixir-versions.yml b/_data/elixir-versions.yml new file mode 100644 index 000000000..468bdabbf --- /dev/null +++ b/_data/elixir-versions.yml @@ -0,0 +1,88 @@ +stable: v1_19 + +v1_19: + name: v1.19 + minimum_otp: 26.0 + recommended_otp: 28.1 + otp_versions: [28, 27, 26] + version: 1.19.4 + +v1_18: + name: v1.18 + minimum_otp: 26.0 + recommended_otp: 27.3.4 + otp_versions: [27, 26, 25] + version: 1.18.4 + +v1_17: + name: v1.17 + minimum_otp: 25.0 + recommended_otp: 27.1.2 + otp_versions: [27, 26, 25] + version: 1.17.3 + +v1_16: + name: v1.16 + minimum_otp: 24.0 + otp_versions: [26, 25, 24] + version: 1.16.3 + +v1_15: + name: v1.15 + minimum_otp: 24.0 + otp_versions: [26, 25, 24] + version: 1.15.8 + +v1_14: + name: v1.14 + minimum_otp: 23.0 + otp_versions: [25, 24, 23] + version: 1.14.5 + +v1_13: + name: v1.13 + minimum_otp: 22.0 + otp_versions: [24, 23, 22] + version: 1.13.4 + +v1_12: + name: v1.12 + minimum_otp: 22.0 + otp_versions: [24, 23, 22] + version: 1.12.3 + +v1_11: + name: v1.11 + minimum_otp: 21.0 + otp_versions: [23, 22, 21] + version: 1.11.4 + +v1_10: + name: v1.10 + minimum_otp: 21.0 + otp_versions: [22, 21] + version: 1.10.4 + +v1_9: + name: v1.9 + minimum_otp: 20.0 + otp_versions: [22, 21, 20] + version: 1.9.4 + +v1_8: + name: v1.8 + minimum_otp: 20.0 + otp_versions: [22, 21, 20] + version: 1.8.2 + +v1_7: + name: v1.7 + minimum_otp: 19.0 + otp_versions: [21, 20, 19] + version: 1.7.4 + +v1_6: + name: v1.6 + minimum_otp: 19.0 + otp_versions: [21, 20, 19] + version: 1.6.6 diff --git a/_data/getting-started.yml b/_data/getting-started.yml deleted file mode 100644 index 875fc16b6..000000000 --- a/_data/getting-started.yml +++ /dev/null @@ -1,112 +0,0 @@ -- title: Getting Started - dir: /getting-started/ - pages: - - title: Introduction - slug: introduction - - - title: Basic types - slug: basic-types - - - title: Basic operators - slug: basic-operators - - - title: Pattern matching - slug: pattern-matching - - - title: case, cond and if - slug: case-cond-and-if - - - title: Binaries, strings and char lists - slug: binaries-strings-and-char-lists - - - title: Keywords and maps - slug: keywords-and-maps - - - title: Modules - slug: modules - - - title: Recursion - slug: recursion - - - title: Enumerables and streams - slug: enumerables-and-streams - - - title: Processes - slug: processes - - - title: IO and the file system - slug: io-and-the-file-system - - - title: alias, require and import - slug: alias-require-and-import - - - title: Module attributes - slug: module-attributes - - - title: Structs - slug: structs - - - title: Protocols - slug: protocols - - - title: Comprehensions - slug: comprehensions - - - title: Sigils - slug: sigils - - - title: try, catch and rescue - slug: try-catch-and-rescue - - - title: Typespecs and behaviours - slug: typespecs-and-behaviours - - - title: Erlang libraries - slug: erlang-libraries - - - title: Where to go next - slug: where-to-go-next - - -- title: Mix and OTP - dir: /getting-started/mix-otp/ - pages: - - title: Introduction to Mix - slug: introduction-to-mix - - - title: Agent - slug: agent - - - title: GenServer - slug: genserver - - - title: Supervisor and Application - slug: supervisor-and-application - - - title: ETS - slug: ets - - - title: Dependencies and umbrella apps - slug: dependencies-and-umbrella-apps - - - title: Task and gen-tcp - slug: task-and-gen-tcp - - - title: Docs, tests and with - slug: docs-tests-and-with - - - title: Distributed tasks and configuration - slug: distributed-tasks-and-configuration - - -- title: Meta-programming in Elixir - dir: /getting-started/meta/ - pages: - - title: Quote and unquote - slug: quote-and-unquote - - - title: Macros - slug: macros - - - title: Domain Specific Languages - slug: domain-specific-languages diff --git a/_includes/bottom.html b/_includes/bottom.html index dc8509af8..bfc4fb987 100644 --- a/_includes/bottom.html +++ b/_includes/bottom.html @@ -2,30 +2,11 @@
- - - - - diff --git a/_includes/categories-list.html b/_includes/categories-list.html index 8056b1d0d..61d4e22c0 100644 --- a/_includes/categories-list.html +++ b/_includes/categories-list.html @@ -1,8 +1,16 @@

Blog Categories

diff --git a/_includes/distilled-by.html b/_includes/distilled-by.html deleted file mode 100644 index 35256a4b9..000000000 --- a/_includes/distilled-by.html +++ /dev/null @@ -1,10 +0,0 @@ -
-

Created at

- - -

- Plataformatec offers consulting and development services for companies using Elixir. -

-
diff --git a/_includes/elixir-radar.html b/_includes/elixir-radar.html deleted file mode 100644 index b4d4090c3..000000000 --- a/_includes/elixir-radar.html +++ /dev/null @@ -1,19 +0,0 @@ -
-

Elixir Radar Newsletter

-

A weekly Elixir email newsletter with content curated by Plataformatec.

-
-
-
- Elixir Radar -
-
- weekly newsletter -
-
-
- - Subscribe now - -
-
-
diff --git a/_includes/events.html b/_includes/events.html new file mode 100644 index 000000000..e8ed467c5 --- /dev/null +++ b/_includes/events.html @@ -0,0 +1,4 @@ + diff --git a/_includes/important-links.html b/_includes/important-links.html index 9b66b07d8..c4d5c0702 100644 --- a/_includes/important-links.html +++ b/_includes/important-links.html @@ -1,40 +1,63 @@
-

Join the Community

+

Important links

-
-

Important links

- +
+ +
+
Watch the Elixir
mini-documentary!
+
+
+{% include events.html %} +
-

Code editor support

+

Join the Community

-{% include elixir-radar.html %} - -{% include distilled-by.html %} +
+ Join the Erlang Ecosystem Foundation +
diff --git a/_includes/mix-otp-preface.html b/_includes/mix-otp-preface.html deleted file mode 100644 index 4459d0d3a..000000000 --- a/_includes/mix-otp-preface.html +++ /dev/null @@ -1,2 +0,0 @@ -This chapter is part of the Mix and OTP guide and it depends on previous chapters in this guide. -For more information, read the introduction guide or check out the chapter index in the sidebar. diff --git a/_includes/pagination.html b/_includes/pagination.html index 79e8af0aa..7e5dd07cb 100644 --- a/_includes/pagination.html +++ b/_includes/pagination.html @@ -7,7 +7,7 @@ {% endif %} {% endif %} - + {% if paginator.page == 1 %} 1 {% else %} diff --git a/_includes/search.html b/_includes/search.html index aabe51dff..9b85975eb 100644 --- a/_includes/search.html +++ b/_includes/search.html @@ -1,15 +1,36 @@ +{% assign stable = site.data.elixir-versions[site.data.elixir-versions.stable].version %} +
-

- News: Announcing GenStage -

+

News: Elixir v1.19.0 released

-
-
- - - -
-
-
\ No newline at end of file + +
diff --git a/_includes/top.html b/_includes/top.html index fe489950c..13ecc8304 100644 --- a/_includes/top.html +++ b/_includes/top.html @@ -1,10 +1,11 @@ - + - - {% if page.title %}{{ page.title }} - {% endif %}Elixir - + + + {% if page.title %}{{ page.title }} - {% endif %}The Elixir programming language + @@ -12,18 +13,12 @@ - + - + + + {% seo title=false %} @@ -32,28 +27,22 @@
diff --git a/_layouts/blog.html b/_layouts/blog.html index 18afcabd8..92a81743d 100644 --- a/_layouts/blog.html +++ b/_layouts/blog.html @@ -7,7 +7,7 @@
diff --git a/_layouts/redirect.html b/_layouts/redirect.html new file mode 100644 index 000000000..464f54e82 --- /dev/null +++ b/_layouts/redirect.html @@ -0,0 +1,14 @@ + + + + + + + + +

Redirecting...

+ Click here if you are not redirected. + + + + diff --git a/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown b/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown index 05d678a0e..591610198 100644 --- a/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown +++ b/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown @@ -1,12 +1,13 @@ --- layout: post title: A peek inside Elixir's Parallel Compiler -author: José Valim +authors: +- José Valim category: Internals -excerpt: Today, a parallel compiler just landed in Elixir master. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. +excerpt: Today, a parallel compiler just landed in Elixir main. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. --- -Today, a parallel compiler just landed in Elixir master. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. +Today, a parallel compiler just landed in Elixir main. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. ## Process-based serial compilation @@ -156,7 +157,7 @@ Notice we added an extra clause to `spawn_compilers` so we can properly handle t :erlang.raise(:error, reason, where) after 10_000 -> - raise "dependency on unexesting module or possible deadlock" + raise "dependency on nonexistent module or possible deadlock" end end diff --git a/_posts/2012-05-25-elixir-v0-5-0-released.markdown b/_posts/2012-05-25-elixir-v0-5-0-released.markdown index 10c023b72..a5701a60d 100644 --- a/_posts/2012-05-25-elixir-v0-5-0-released.markdown +++ b/_posts/2012-05-25-elixir-v0-5-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.5.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: We have finally released Elixir v0.5.0! This marks the first release since the language was rewritten. In this blog post, we will discuss what we achieved during this time and what are the next steps! @@ -9,7 +10,7 @@ excerpt: We have finally released Elixir v0.5.0! This marks the first release si We have finally released [Elixir](/) v0.5.0! This marks the first release since the language was rewritten. In this blog post, we will discuss what we achieved during this time and what are the next steps! -If you don't care about any of these, you can go straight to our [Getting Started guide](/getting-started/introduction.html). If you do, keep on reading! +If you don't care about any of these, you can go straight to our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html). If you do, keep on reading! ## Looking back @@ -27,9 +28,9 @@ Around February of that year, feeling confident enough about the direction the l One of the goals we have set was to have a good website and documentation before the next official release. With the help of the Plataformatec team, we created a logo for Elixir and put this website live. -At the same time, [we were working on pygments support](https://bitbucket.org/birkenfeld/pygments-main/pull-request/57/add-elixir-and-elixir-console-lexers), a [documentation generation tool](https://github.com/elixir-lang/ex_doc) and many others. Soon, Github was able to syntax highlight Elixir code and [our API documentation was online](/). +At the same time, [we were working on pygments support](https://bitbucket.org/birkenfeld/pygments-main/pull-request/57/add-elixir-and-elixir-console-lexers), a [documentation generation tool](https://github.com/elixir-lang/ex_doc) and many others. Soon, GitHub was able to syntax highlight Elixir code and [our API documentation was online](/). -At the same time, people started to gather around #elixir-lang channel on irc.freenode.net and [play with Elixir](https://github.com/elixir-lang/elixir/tree/master/lib/mix), [start their](https://github.com/guedes/exdate) [own projects](https://github.com/yrashk/validatex) and [tutorials](https://github.com/alco/elixir/wiki/Erlang-Syntax:-A-Crash-Course). +At the same time, people started to gather around #elixir-lang channel on irc.freenode.net (now migrated to #elixir on irc.libera.chat) to [play with Elixir](https://github.com/elixir-lang/elixir/tree/main/lib/mix), [start their](https://github.com/guedes/exdate) [own projects](https://github.com/yrashk/validatex) and [tutorials](https://github.com/alco/elixir/wiki/Erlang-Syntax:-A-Crash-Course). Although the initial release was scheduled to April 2012, the feedback from such early developers forced us to review some design and syntax decisions and were extremely important to shape the language as it is today. @@ -41,8 +42,8 @@ There are still many, many things to do! In the next months, we will continue wo We will also work on better integration and documentation on building Erlang systems. Erlang ships with the [Open Telecom Platform](https://en.wikipedia.org/wiki/Open_Telecom_Platform) which provides many tools to build distributed applications. In v0.5.0, all these tools are already available but we want to make the build process even simpler. -In parallel, we will improve our [documentation generation tool](https://github.com/elixir-lang/ex_doc) and [build tool](https://github.com/elixir-lang/elixir/tree/master/lib/mix) which will likely be merged into core when they are solid enough. +In parallel, we will improve our [documentation generation tool](https://github.com/elixir-lang/ex_doc) and [build tool](https://github.com/elixir-lang/elixir/tree/main/lib/mix) which will likely be merged into core when they are solid enough. Finally, we will continue improving the Standard Library. Although Elixir's goal is to rely on Erlang the most as possible, we also want to provide a small Standard Library which makes better use of Elixir semantics. For the next weeks, we will focus on improving the IO and File manipulation modules. New data types may also appear, for example, ranges come to my mind. -Check out our [home page](/) and the [getting started guide](/getting-started/introduction.html) for more information. Welcome aboard and grab a cup of Elixir, because you are certainly going to enjoy the ride! +Check out our [home page](/) and the [getting started guide](https://hexdocs.pm/elixir/introduction.html) for more information. Welcome aboard and grab a cup of Elixir, because you are certainly going to enjoy the ride! diff --git a/_posts/2012-08-01-elixir-v0-6-0-released.markdown b/_posts/2012-08-01-elixir-v0-6-0-released.markdown index 3453a719b..becafdf7c 100644 --- a/_posts/2012-08-01-elixir-v0-6-0-released.markdown +++ b/_posts/2012-08-01-elixir-v0-6-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.6.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: We have finally released Elixir v0.6.0! This release includes a build tool called Mix, support for Erlang typespecs, many improvements to IEx and improved IO, File and Macro support. @@ -19,9 +20,9 @@ When [we released version v0.5.0](/blog/2012/05/25/elixir-v0-5-0-released/), we Our interactive shell (IEx) also had many improvements, thanks to the Elixir developer community. We now have easy access to documentation, remote shells, autocomplete and much more. In order to show you a bit of what you can do in this release, we have prepared a short (~6 min) screencast: - + Elixir v0.6 quick tour - Mix and IEx from Plataformatec on Vimeo. That's it. For the next months, we will continue improving Elixir (you can see some ideas floating around in the [issues tracker](https://github.com/elixir-lang/elixir/issues)) but we will start to focus on other tools and libraries for the community. -Thank you and don't forget to [give Elixir a try](/getting-started/introduction.html)! +Thank you and don't forget to [give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2012-10-20-elixir-v0-7-0-released.markdown b/_posts/2012-10-20-elixir-v0-7-0-released.markdown index 16db42d47..d2f181612 100644 --- a/_posts/2012-10-20-elixir-v0-7-0-released.markdown +++ b/_posts/2012-10-20-elixir-v0-7-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.7.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.7.0 is released with many improvements! Read on for more information. @@ -15,4 +16,4 @@ This release also includes some backwards incompatible changes, but the majority For more information, read out the [CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.7.0/CHANGELOG.md). -Thank you and don't forget to [give Elixir a try](/getting-started/introduction.html)! +Thank you and don't forget to [give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2012-11-18-elixir-v0-7-1-released.markdown b/_posts/2012-11-18-elixir-v0-7-1-released.markdown index 75e557cb4..113bcd31f 100644 --- a/_posts/2012-11-18-elixir-v0-7-1-released.markdown +++ b/_posts/2012-11-18-elixir-v0-7-1-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.7.1 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.7.1 was released to celebrate the end of a two months journey traveling around Europe, United States and Brazil talking about Elixir. @@ -11,11 +12,11 @@ Elixir v0.7.1 was released this weekend to celebrate the end of a two months jou This is a minor release that contains a couple enhancements regarding UTF-8, [dialyzer](http://www.erlang.org/doc/man/dialyzer.html) support and bug fixes. -During this time traveling around, we have spoken at many conferences, as [Strange Loop](http://thestrangeloop.com/), [Øredev](http://oredev.org/), [QCon SP](http://qconsp.com/) and [Rupy](http://rupy.eu/) as well as at different companies. Developers from different backgrounds have shown interest in Elixir, [written about it](http://spin.atomicobject.com/2012/10/31/elixir-erlang-and-the-dining-philosophers/), joined us at #elixir-lang on freenode and contributed to the language. As of today, Elixir is powered by 51 different contributors! +During this time traveling around, we have spoken at many conferences, as [Strange Loop](http://thestrangeloop.com/), [Øredev](http://oredev.org/), [QCon SP](http://qconsp.com/) and [Rupy](http://rupy.eu/) as well as at different companies. Developers from different backgrounds have shown interest in Elixir, [written about it](http://spin.atomicobject.com/2012/10/31/elixir-erlang-and-the-dining-philosophers/), and contributed to the language. As of today, Elixir is powered by 51 different contributors! In case you missed any of those conferences, [the talk I presented at Øredev is available and you can watch it now](https://vimeo.com/53221562). The slides are also available below. -If you want to hear more about Elixir at a conference or an event, please let us know. Thank you and don't forget to [give Elixir a try](/getting-started/introduction.html)! +If you want to hear more about Elixir at a conference or an event, please let us know. Thank you and don't forget to [give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2012-12-04-elixir-v0-7-2-released.markdown b/_posts/2012-12-04-elixir-v0-7-2-released.markdown index 7aba55020..b4ec972c7 100644 --- a/_posts/2012-12-04-elixir-v0-7-2-released.markdown +++ b/_posts/2012-12-04-elixir-v0-7-2-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.7.2 released -author: Yurii Rashkovskii +authors: +- Yurii Rashkovskii category: Releases excerpt: Elixir v0.7.2 is released, new, improved type specifications syntax and many other improvements. @@ -44,10 +45,10 @@ We've also added a more compact and visual form of the `function` helper. Now, instead of `function(Enum, :all?, 2)` you can use `function(Enum.all?/2)`. We've also figured out how to achieve an up to 6x [performance increase](https://github.com/elixir-lang/elixir/blob/v0.7.2/lib/elixir/lib/kernel.ex#L1386-L1417) -under some circunstances when using records. +under some circumstances when using records. ...and [many other fixes & improvements](https://github.com/elixir-lang/elixir/blob/v0.7.2/CHANGELOG.md). Lastly, but not least importantly, I'd like to mention that we're very excited about how the community around Elixir is building up. Thank you all for being around and supporting us! -[Learn more about Elixir](/getting-started/introduction.html)! +[Learn more about Elixir](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2013-01-27-elixir-v0-8-0-released.markdown b/_posts/2013-01-27-elixir-v0-8-0-released.markdown index aae6f2db7..9f6236b10 100644 --- a/_posts/2013-01-27-elixir-v0-8-0-released.markdown +++ b/_posts/2013-01-27-elixir-v0-8-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.8.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: On the last 9th January, we celebrated two years since Elixir's first commit and to celebrate this occasion we have prepared a big release. Elixir v0.8 is out, with documentation, optimizations, bug fixes and shiny new features. Let's take a look at them! --- @@ -18,7 +19,7 @@ And applications can be started directly from the command line as well: elixir --app my_app -We have written a whole [guide chapter about creating OTP applications, supervisors and servers](/getting-started/mix-otp/supervisor-and-application.html). Give it a try! +We have written a whole [guide chapter about creating OTP applications, supervisors and servers](https://hexdocs.pm/elixir/supervisor-and-application.html). Give it a try! ## Improved Unicode support @@ -70,15 +71,15 @@ When some code call the `unless` macro above, in previous Elixir versions, it wo Elixir v0.8 ensures that the `unless` macro above will expand to the same `if` macro available when quoted, guaranteeing different libraries can integrate easily without imposing hidden requirements. -You can read more about [macros in the getting started guide](/getting-started/case-cond-and-if.html) or [go deep into the quote macro docs](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#quote/2). +You can read more about [macros in the getting started guide](https://hexdocs.pm/elixir/case-cond-and-if.html) or [go deep into the quote macro docs](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#quote/2). ## A new way to manipulate pathnames -Elixir v0.8 contains a bit of house cleaning too. We have created [the Path module](https://hexdocs.pm/elixir/Path.html) to accommodate functions used to manipulate filesystem paths and have also added functions like [`System.tmp_dir` and `System.user_home`](https://hexdocs.pm/elixir/System.html) which are meant to work accross different operating systems and are very handy when scripting. +Elixir v0.8 contains a bit of house cleaning too. We have created [the Path module](https://hexdocs.pm/elixir/Path.html) to accommodate functions used to manipulate filesystem paths and have also added functions like [`System.tmp_dir` and `System.user_home`](https://hexdocs.pm/elixir/System.html) which are meant to work across different operating systems and are very handy when scripting. ## The new HashDict -For last but not least, Elixir ships with a [new HashDict implementation](https://github.com/elixir-lang/elixir/blob/master/lib/elixir/lib/hash_dict.ex). In Erlang, there are different key-value store implementations and often you need to pick which one is the best for you based on the average size of the dictionary. Generally speaking, [orddicts](http://www.erlang.org/doc/man/orddict.html) are efficient and fast when you want to hold a handful of items, otherwise you should consider [gb_trees](http://www.erlang.org/doc/man/gb_trees.html) unless you want to hold thousands of items, when then [dict](http://www.erlang.org/doc/man/dict.html) becomes your best option. The fact those implementations do not provide the same API, makes it harder to change your code when you realize another implementation would be better fit. +For last but not least, Elixir ships with a [new HashDict implementation](https://github.com/elixir-lang/elixir/blob/main/lib/elixir/lib/hash_dict.ex). In Erlang, there are different key-value store implementations and often you need to pick which one is the best for you based on the average size of the dictionary. Generally speaking, [orddicts](http://www.erlang.org/doc/man/orddict.html) are efficient and fast when you want to hold a handful of items, otherwise you should consider [gb_trees](http://www.erlang.org/doc/man/gb_trees.html) unless you want to hold thousands of items, when then [dict](http://www.erlang.org/doc/man/dict.html) becomes your best option. The fact those implementations do not provide the same API, makes it harder to change your code when you realize another implementation would be better fit. For Elixir, we decided to have a single dictionary implementation that would scale as needed. It would start as a compact representation for a handful of items and expand and rehash accordingly as new items are added or removed, providing fast access and modification times on all ranges. We are glad to say our goals were reached and a new `HashDict` implementation ships with Elixir v0.8. @@ -100,4 +101,4 @@ We continue actively working on Elixir and this release is the [result of our ef Also, we previously announced Elixir is going to be released frequently, every 2 to 4 weeks. We have made a small detour to get v0.8.0 out of the door, but we are back to our regular schedule as of today! -[Celebrate with us and give Elixir a try](/getting-started/introduction.html)! +[Celebrate with us and give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2013-04-19-google-summer-of-code-2013.markdown b/_posts/2013-04-19-google-summer-of-code-2013.markdown index b025579f6..d964d7c1b 100644 --- a/_posts/2013-04-19-google-summer-of-code-2013.markdown +++ b/_posts/2013-04-19-google-summer-of-code-2013.markdown @@ -1,7 +1,8 @@ --- layout: post title: Google Summer of Code 2013 -author: José Valim +authors: +- José Valim category: Announcements excerpt: Elixir is taking part in Google Summer of Code 2013! Are you a student? Join us! --- diff --git a/_posts/2013-04-29-elixir-v0-8-2-released.markdown b/_posts/2013-04-29-elixir-v0-8-2-released.markdown index 9579766ac..fea64d566 100644 --- a/_posts/2013-04-29-elixir-v0-8-2-released.markdown +++ b/_posts/2013-04-29-elixir-v0-8-2-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.8.2 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.8.2 is released with bug fixes, better Erlang R16 support and doctests. --- @@ -46,4 +47,4 @@ end You can learn more about [doctests on our documentation page](https://hexdocs.pm/ex_unit/ExUnit.DocTest.html) and get more information about our latest release [on the CHANGELOG](https://github.com/elixir-lang/elixir/blob/ed27611f48ba150404c95fe15f1d6058a4287330/CHANGELOG.md). -If you are new to Elixir, [it's easy to get started with](/getting-started/introduction.html)! +If you are new to Elixir, [it's easy to get started with](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2013-05-02-elixir-on-xen.markdown b/_posts/2013-05-02-elixir-on-xen.markdown index a41c5a3de..9f1de4c2b 100644 --- a/_posts/2013-05-02-elixir-on-xen.markdown +++ b/_posts/2013-05-02-elixir-on-xen.markdown @@ -1,18 +1,19 @@ --- layout: post title: Elixir on Xen -author: José Valim +authors: +- José Valim category: Announcements excerpt: The Erlang on Xen team has added support for Elixir and we will tell you how you can use it! --- -Elixir uses Erlang underneath, all the way down. Thanks to this, an Elixir project can run on the recently revealed “OS-less” Erlang VM called LING VM. LING VM is the core technology of [Erlang on Xen](http://erlangonxen.org). +Elixir uses Erlang underneath, all the way down. Thanks to this, an Elixir project can run on the recently revealed "OS-less" Erlang VM called LING VM. LING VM is the core technology of [Erlang on Xen](http://erlangonxen.org). ## Why Xen? [Xen](https://en.wikipedia.org/wiki/Xen) is an open-source baremetal hypervisor that allows many operating systems to run on the same hardware. Xen is frequently used for server virtualization, Infrastructure as a Service (IaaS) and security applications. -Elixir on Xen runs on top of the Xen Hypervisor (via the LING VM) but with no traditional OS underneath it, taking away numerous administrative, scalability, and performance issues. This limits options of a malicious attacker, making it an excellent choice for high-security applications, and reduces startup latency, allowing developers to spawn new VMs in less than 100 miliseconds. +Elixir on Xen runs on top of the Xen Hypervisor (via the LING VM) but with no traditional OS underneath it, taking away numerous administrative, scalability, and performance issues. This limits options of a malicious attacker, making it an excellent choice for high-security applications, and reduces startup latency, allowing developers to spawn new VMs in less than 100 milliseconds. You can learn more about Xen and the LING VM on the [Erlang on Xen website](http://erlangonxen.org). diff --git a/_posts/2013-05-23-elixir-v0-9-0-released.markdown b/_posts/2013-05-23-elixir-v0-9-0-released.markdown index cd319e93a..edee304e1 100644 --- a/_posts/2013-05-23-elixir-v0-9-0-released.markdown +++ b/_posts/2013-05-23-elixir-v0-9-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.9.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.9.0 is released with support for reducers, umbrella projects, faster compilation times and dropped support for R15 and earlier OTP versions. --- @@ -77,8 +78,8 @@ defimpl Enumerable, for: List do do_reduce(list, acc, fun) end - defp do_reduce([h | t], acc, fun) do - do_reduce(t, fun.(h, acc), fun) + defp do_reduce([head | tail], acc, fun) do + do_reduce(tail, fun.(head, acc), fun) end defp do_reduce([], acc, fun) do @@ -123,11 +124,11 @@ A special thanks to [Eric Meadows-Jonsson](https://github.com/ericmj) for implem We have also many other smaller improvements: * Our CLI now supports `--hidden` and `--cookie` flags which are useful for distributed modes; -* Our test framework, ExUnit, is now able to capture all the communication that happens with a registed IO device, like `:stdio` and `:stderr`, via [`ExUnit.CaptureIO`](https://hexdocs.pm/ex_unit/ExUnit.CaptureIO.html). This is very useful for testing how your software reacts to some inputs and what it prints to the terminal; +* Our test framework, ExUnit, is now able to capture all the communication that happens with a registered IO device, like `:stdio` and `:stderr`, via [`ExUnit.CaptureIO`](https://hexdocs.pm/ex_unit/ExUnit.CaptureIO.html). This is very useful for testing how your software reacts to some inputs and what it prints to the terminal; * `IEx` now allows files to be imported into the shell with `import_file` and also loads `~/.iex` on startup for custom configuration; * The `String`, `Enum` and `Dict` modules got more convenience functions that goes from checking unicode character validity to taking values out of a dictionary; * And many, many more! A huge thank you to our community for sending bug reports, providing bug fixes and contributing all those amazing features. And when are **you** joining us? :) -Give Elixir a try! You can start with our [getting started guide](/getting-started/introduction.html), or [check this 30 minute video from PragProg](https://www.youtube.com/watch?v=a-off4Vznjs&feature=youtu.be) or buy the beta version of [Programming Elixir](https://pragprog.com/book/elixir/programming-elixir). +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or [check this 30 minute video from PragProg](https://www.youtube.com/watch?v=a-off4Vznjs&feature=youtu.be) or buy the beta version of [Programming Elixir](https://pragprog.com/book/elixir/programming-elixir). diff --git a/_posts/2013-07-13-elixir-v0-10-0-released.markdown b/_posts/2013-07-13-elixir-v0-10-0-released.markdown index 3d28ad7ba..c00858ea8 100644 --- a/_posts/2013-07-13-elixir-v0-10-0-released.markdown +++ b/_posts/2013-07-13-elixir-v0-10-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.10.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.10.0 is out with support for streams, sets, pretty printing and many improvements for Mix and ExUnit. --- @@ -82,4 +83,4 @@ Other notable improvements are: There are also improvements to typespecs, error messages, many bug fixes and some backwards incompatible changes. We have posted a detailed [upgrade instructions on the mailing list](https://groups.google.com/forum/?fromgroups#!topic/elixir-lang-talk/ksrefrgK1eY). For a general overview, [check out the CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.10.0/CHANGELOG.md). -Give Elixir a try! You can start with our [getting started guide](/getting-started/introduction.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. diff --git a/_posts/2013-08-08-elixir-design-goals.markdown b/_posts/2013-08-08-elixir-design-goals.markdown index 6b5b316e8..5959dab92 100644 --- a/_posts/2013-08-08-elixir-design-goals.markdown +++ b/_posts/2013-08-08-elixir-design-goals.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir Design Goals -author: José Valim +authors: +- José Valim category: Internals excerpt: Highlight of Elixir design goals. --- @@ -125,7 +126,7 @@ defmodule Hello, do: ( ) ``` -And finally we added `do/end` as convenience for the common `do: (...)` construct: +And finally we added `do`-`end` as convenience for the common `do: (...)` construct: ```elixir defmodule Hello do @@ -136,7 +137,7 @@ defmodule Hello do end ``` -Given my previous background in Ruby, it is natural that some of the constructs added were borrowed from Ruby. However, those additions were a by-product, never a language goal. +Given my previous background in Ruby, it is natural that some of the constructs added were borrowed from Ruby. However, those additions were a by-product, and not a language goal. Many language constructs are also inspired by their Erlang counter-parts, like some of the control-flow macros, operators and containers. Notice how some Elixir code: @@ -208,8 +209,8 @@ There are many other protocols exposed by the language, like [the `Inspect` prot ## Summing up -The goal of this post was to sumarize the language goals: compatibility, productivity and extensibility. By being compatibile with the Erlang VM, we are providing developers another toolset for building concurrent, distributed and fault-tolerant systems. +The goal of this post was to sumarize the language goals: compatibility, productivity and extensibility. By being compatible with the Erlang VM, we are providing developers another toolset for building concurrent, distributed and fault-tolerant systems. We also hope to have clarified what Elixir brings to the Erlang VM, in particular, meta-programming through macros, polymorphic constructs for extensibility and a data-focused standard library with extensible and consistent APIs for diverse types, including strict and lazy enumeration, unicode handling, a test framework and more. -Give Elixir a try! You can start with our [getting started guide](/getting-started/introduction.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. diff --git a/_posts/2013-11-05-elixir-v0-11-0-released.markdown b/_posts/2013-11-05-elixir-v0-11-0-released.markdown index 80e3bb825..1a668262c 100644 --- a/_posts/2013-11-05-elixir-v0-11-0-released.markdown +++ b/_posts/2013-11-05-elixir-v0-11-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.11.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.11.0 is out and it focus on improving and optimizing the patterns often used by the community. --- @@ -67,6 +68,6 @@ In the optimization front, we have pushed the first iteration of a [feature call And finally, a minor but frequently asked feature has finally arrived into Elixir: variables follow the same rules as other identifiers in the language, which means developers can now name their variables `is_atom?`. For a general overview, [check out the CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.11.0/CHANGELOG.md). -Give Elixir a try! You can start with our [getting started guide](/getting-started/introduction.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. **PS:** We have just released v0.11.1 which addresses a regression in Mix and improves the dependencies update process. diff --git a/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown b/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown index 9a0670d9f..b29807cb4 100644 --- a/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown +++ b/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir's new continuable enumerators -author: Peter Minten +authors: +- Peter Minten category: Internals excerpt: In 0.12.0 Elixir's enumerators have gained the ability to suspend value production and to terminate early. diff --git a/_posts/2013-12-15-elixir-v0-12-0-released.markdown b/_posts/2013-12-15-elixir-v0-12-0-released.markdown index a6a699a49..fe8a349e3 100644 --- a/_posts/2013-12-15-elixir-v0-12-0-released.markdown +++ b/_posts/2013-12-15-elixir-v0-12-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.12.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.12.0 is out with improved enumerables, build patterns and welcoming a new member to our team --- @@ -18,7 +19,7 @@ Although most users don't need to concern with the implementation of the Enumera ## Mix -The tool that received most improvements in this release was Mix. The biggest change is that Mix no longer compiles projects in place but to the `_build` directory. For example, take the [Ecto project](https://github.com/elixir-lang/ecto) that [depends on `postgrex` and `poolboy`](https://github.com/elixir-lang/ecto/blob/master/mix.exs#L24-L25). When compiled, all the artifacts will be placed in the `_build` directory like this: +The tool that received most improvements in this release was Mix. The biggest change is that Mix no longer compiles projects in place but to the `_build` directory. For example, take the [Ecto project](https://github.com/elixir-ecto/ecto) that [depends on `postgrex` and `poolboy`](https://github.com/elixir-ecto/ecto/blob/v0.1.0/mix.exs#L25-L30). When compiled, all the artifacts will be placed in the `_build` directory like this: ``` _build @@ -43,10 +44,10 @@ Mix has also added support to optional dependencies and improved common patterns With this release, we also want to welcome [Eric MJ](https://github.com/ericmj) to the Elixir Team. He has done fantastic work on Elixir, helping us maintain the codebase and working on many of the important features from previous releases and now many more to come. -Eric is also maintainer of both [Ecto](https://github.com/elixir-lang/ecto) and [Postgrex](https://github.com/ericmj/postgrex) projects. Which are proving to be very useful to the Elixir community too! +Eric is also maintainer of both [Ecto](https://github.com/elixir-ecto/ecto) and [Postgrex](https://github.com/elixir-ecto/postgrex) projects. Which are proving to be very useful to the Elixir community too! ## Tidying up There were other small changes, like additions to the `Float` module and improvements the to the typespec syntax. To see the full list, please [see the CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.12.0/CHANGELOG.md). -Give Elixir a try! You can start with our [getting started guide](/getting-started/introduction.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. diff --git a/_posts/2014-04-21-elixir-v0-13-0-released.markdown b/_posts/2014-04-21-elixir-v0-13-0-released.markdown index 554029398..995e4d68c 100644 --- a/_posts/2014-04-21-elixir-v0-13-0-released.markdown +++ b/_posts/2014-04-21-elixir-v0-13-0-released.markdown @@ -1,9 +1,10 @@ --- layout: post title: Elixir v0.13.0 released, hex.pm and ElixirConf announced -author: José Valim +authors: +- José Valim category: Releases -excerpt: "Elixir v0.13.0 comes with substantial improvements to the language: maps, structs, comprehensiona and more. It also marks the announcement of the hex.pm package manager and the announcment of ElixirConf!" +excerpt: "Elixir v0.13.0 comes with substantial improvements to the language: maps, structs, comprehensiona and more. It also marks the announcement of the hex.pm package manager and the announcement of ElixirConf!" --- Hello folks! @@ -28,7 +29,7 @@ In a nutshell, here is what new: * Elixir v0.13 also provides structs, an alternative to Elixir records. Structs are more flexible than records, provide faster polymorphic operations, and still provide the same compile-time guarantees many came to love in records; -* The [Getting Started guide](/getting-started/introduction.html) was rewritten from scratch. The previous guide was comprised of 7 chapters and was about to become 2 years old. The new guide features 20 chapters, it explores the new maps and structs (which are part of this release), and it goes deeper into topics like IO and File handling. It also includes an extra guide, still in development, about [Meta-Programming in Elixir](/getting-started/meta/quote-and-unquote.html); +* The [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) was rewritten from scratch. The previous guide was comprised of 7 chapters and was about to become 2 years old. The new guide features 20 chapters, it explores the new maps and structs (which are part of this release), and it goes deeper into topics like IO and File handling. It also includes an extra guide, still in development, about [Meta-Programming in Elixir](https://hexdocs.pm/elixir/quote-and-unquote.html); * Elixir v0.13 provides a new comprehension syntax that not only works with lists, but with any [`Enumerable`](https://hexdocs.pm/elixir/Enumerable.html). The output of a comprehension is also extensible via the [`Collectable`](https://hexdocs.pm/elixir/Collectable.html) protocol; @@ -42,7 +43,7 @@ Even with all those improvements, Elixir v0.13.0 is backwards compatible with El Maps are key-value data structures: -```iex +```elixir iex> map = %{"hello" => :world} %{"hello" => :world} iex> map["hello"] @@ -55,7 +56,7 @@ Maps do not have a explicit ordering and keys and values can be any term. Maps can be pattern matched on: -```iex +```elixir iex> %{"hello" => world} = map %{"hello" => :world} iex> world @@ -68,11 +69,11 @@ iex> %{"other" => value} = map A map pattern will match any map that has all the keys specified in the pattern. The values for the matching keys must also match. For example, `%{"hello" => world}` will match any map that has the key `"hello"` and assign the value to `world`, while `%{"hello" => "world"}` will match any map that has the key `"hello"` with value equals to `"world"`. An empty map pattern (`%{}`) will match all maps. -Developers can use the functions in the [`Map` module](https://hexdocs.pm/elixir/Map.html) to work with maps. For more information on maps and how they compare to other associative data structures in the language, please check the [Maps chapter in our new Getting Started guide](/getting-started/keywords-and-maps.html). Elixir Sips has also released two episodes that cover maps ([part 1](http://elixirsips.com/episodes/054_maps_part_1.html) and [part 2](http://elixirsips.com/episodes/055_maps_part_2.html)). +Developers can use the functions in the [`Map` module](https://hexdocs.pm/elixir/Map.html) to work with maps. For more information on maps and how they compare to other associative data structures in the language, please check the [Maps chapter in our new Getting Started guide](https://hexdocs.pm/elixir/keywords-and-maps.html). Elixir Sips has also released two episodes that cover maps ([part 1](http://elixirsips.com/episodes/054_maps_part_1.html) and [part 2](http://elixirsips.com/episodes/055_maps_part_2.html)). Maps also provide special syntax for creating, accessing and updating maps with atom keys: -```iex +```elixir iex> user = %{name: "john", age: 27} %{name: "john", age: 27} iex> user.name @@ -85,7 +86,7 @@ iex> user.name Both access and update syntax above expect the given keys to exist. Trying to access or update a key that does not exist raises an error: -```iex +```elixir iex> %{ user | address: [] } ** (ArgumentError) argument error :maps.update(:address, [], %{}) @@ -110,7 +111,7 @@ Internally, this record is represented as the following tuple: Records can also be created and pattern matched on: -```iex +```elixir iex> user = User[name: "john"] User[name: "john", age: 0] iex> user.name @@ -156,21 +157,21 @@ end Now a `User` struct can be created without a need to explicitly list all necessary fields: -```iex +```elixir iex> user = %User{name: "john"} %User{name: "john", age: 0} ``` Trying to create a struct with an unknown key raises an error during compilation: -```iex +```elixir iex> user = %User{address: []} ** (CompileError) unknown key :address for struct User ``` Furthermore, every struct has a `__struct__` field which contains the struct name: -```iex +```elixir iex> user.__struct__ User ``` @@ -179,7 +180,7 @@ The `__struct__` field is also used for polymorphic dispatch in protocols, addre It is interesting to note that structs solve both drawbacks we have earlier mentioned regarding records. Structs are purely data and polymorphic dispatch is now faster and more robust as it happens only for explicitly tagged structs. -For more information on structs, check out the [Structs chapter in the getting started guide](/getting-started/structs.html) (you may also want to read the new [Protocols chapter](/getting-started/protocols.html) after it). +For more information on structs, check out the [Structs chapter in the getting started guide](https://hexdocs.pm/elixir/structs.html) (you may also want to read the new [Protocols chapter](https://hexdocs.pm/elixir/protocols.html) after it). ## Maps, structs and the future @@ -187,7 +188,7 @@ With the introduction of maps and structs, some deprecations will arrive on upco Note though only Elixir records are being deprecated. Erlang records, which are basically syntax sugar around tuples, will remain in the language for the rare cases Elixir developers need to interact with Erlang libraries that provide records. In particular, the [Record](https://hexdocs.pm/elixir/Record.html) has been updated to provide the new Record API (while keeping the old one for backwards compatibility). -Finally, structs are still in active development and new features, like `@derive`, should land in upcoming Elixir releases. For those interested, the [original maps and structs proposal is still availble](https://gist.github.com/josevalim/b30c881df36801611d13). +Finally, structs are still in active development and new features, like `@derive`, should land in upcoming Elixir releases. For those interested, the [original maps and structs proposal is still available](https://gist.github.com/josevalim/b30c881df36801611d13). ## Comprehensions @@ -195,21 +196,21 @@ Erlang R17 also introduced recursion to anonymous functions. This feature, while The most common use case of a comprehension are [list comprehensions](https://en.wikipedia.org/wiki/List_comprehension). For example, we can get all the square values of elements in a list as follows: -```iex +```elixir iex> for n <- [1, 2, 3, 4], do: n * n [1, 4, 9, 16] ``` We say the `n <- [1, 2, 3, 4]` part is a comprehension generator. In previous Elixir versions, Elixir supported only lists in generators. In Elixir v0.13.0, any Enumerable is supported (ranges, maps, etc): -```iex +```elixir iex> for n <- 1..4, do: n * n [1, 4, 9, 16] ``` As in previous Elixir versions, there is also support for a bitstring generator. In the example below, we receive a stream of RGB pixels as a binary and break it down into triplets: -```iex +```elixir iex> pixels = <<213, 45, 132, 64, 76, 32, 76, 0, 0, 234, 32, 15>> iex> for <>, do: {r, g, b} [{213, 45, 132}, {64, 76, 32}, {76, 0, 0}, {234, 32, 15}] @@ -217,7 +218,7 @@ iex> for <>, do: {r, g, b} By default, a comprehension returns a list as a result. However the result of a comprehension can be inserted into different data structures by passing the `:into` option. For example, we can use bitstring generators with the `:into` option to easily remove all spaces in a string: -```iex +```elixir iex> for <>, c != ?\s, into: "", do: <> "helloworld" ``` @@ -226,7 +227,7 @@ Sets, maps and other dictionaries can also be given with the `:into` option. In For example, the `IO` module provides streams, that are both `Enumerable` and `Collectable`. You can implement an echo terminal that returns whatever is typed into the shell, but in upcase, using comprehensions: -```iex +```elixir iex> stream = IO.stream(:stdio, :line) iex> for line <- stream, into: stream do ...> String.upcase(line) <> "\n" @@ -248,7 +249,7 @@ $ MIX_ENV=test mix deps.get $ mix test ``` -In Elixir v0.13, `mix deps.get` only fetches dependencies and it does so accross all environments (unless an `--only` flag is specified). To support this new behaviour, dependencies now support the `:only` option: +In Elixir v0.13, `mix deps.get` only fetches dependencies and it does so across all environments (unless an `--only` flag is specified). To support this new behaviour, dependencies now support the `:only` option: ```elixir def deps do @@ -279,4 +280,4 @@ That said, in the next months we plan to: * Study how patterns like tasks and agents can be integrated into the language, often picking up the lessons learned by libraries like [e2](http://e2project.org/erlang.html) and [functionality exposed by OTP itself](http://www.erlang.org/doc/man/rpc.html); * Rewrite the Mix and ExUnit guides to focus on applications and OTP as a whole, rebranding it to "Building Apps with Mix and OTP"; -You can learn more about Elixir in our [Getting Started guide](/getting-started/introduction.html) and download this release in the [v0.13 announcement](https://github.com/elixir-lang/elixir/releases/tag/v0.13.0). We hope to see you at [ElixirConf](http://elixirconf.com/) as well as pushing your packages to [hex.pm](https://hex.pm/). +You can learn more about Elixir in our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) and download this release in the [v0.13 announcement](https://github.com/elixir-lang/elixir/releases/tag/v0.13.0). We hope to see you at [ElixirConf](http://elixirconf.com/) as well as pushing your packages to [hex.pm](https://hex.pm/). diff --git a/_posts/2014-06-17-elixir-v0-14-0-released.markdown b/_posts/2014-06-17-elixir-v0-14-0-released.markdown index b1955d87b..cfa23e5ca 100644 --- a/_posts/2014-06-17-elixir-v0-14-0-released.markdown +++ b/_posts/2014-06-17-elixir-v0-14-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.14.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: "Elixir v0.14.0 is released and expands the work on structs and bringing more of OTP right into Elixir and Mix" --- @@ -144,9 +145,9 @@ In v0.14.0, Elixir closely integrates with OTP by providing modules for building We have also introduced the concepts of [agents](https://hexdocs.pm/elixir/Agent.html) and the idea of [tasks](https://hexdocs.pm/elixir/Task.html), which can be supervised and distributed. Application configuration has been made first class in Mix, allowing developers to configure their dependencies, sometimes even using different configurations per environment (dev, test or prod by default). -This functionality is at the core of building applications in Erlang and Elixir. For this reason we have published a new guide called [Mix and OTP](/getting-started/mix-otp/introduction-to-mix.html) where we build a distributed key-value store to help explore all concepts mentioned above. The guide is quite fresh, so please do submit pull requests for typos and mistakes. Feedback is also welcome! +This functionality is at the core of building applications in Erlang and Elixir. For this reason we have published a new guide called [Mix and OTP](https://hexdocs.pm/elixir/introduction-to-mix.html) where we build a distributed key-value store to help explore all concepts mentioned above. The guide is quite fresh, so please do submit pull requests for typos and mistakes. Feedback is also welcome! -Note "Mix and OTP" is our most advanced guide so far and it expects you to have read our introductory guide. In case you haven't yet, you can [get started here](/getting-started/introduction.html). +Note "Mix and OTP" is our most advanced guide so far and it expects you to have read our introductory guide. In case you haven't yet, you can [get started here](https://hexdocs.pm/elixir/introduction.html). ## What's next? @@ -160,6 +161,6 @@ With v0.14.0 we have reached many of the milestones [we have set in the previous * Solve all remaining [open issues](https://github.com/elixir-lang/elixir/issues?state=open). We have always kept the issues tracker tidy and there is little work left to solve the existing issues. Note we have also listed all [upcoming backwards incompatible changes](https://github.com/elixir-lang/elixir/issues?labels=Note%3ABackwards+incompatible&page=1&state=open). Many of those changes will actually be deprecated first and developers should be able to follow along without breaking changes in minor releases, but they are breaking changes in the sense they work in v0.14.0 but will work differently by the time v1.0 is released; -That's all for now! Elixir developers can see [a summary of all changes in v0.14.0 in the release notes](https://github.com/elixir-lang/elixir/releases/tag/v0.14.0). In case you are new around here, you can get started with Elixir by reading [our Getting Started guide](/getting-started/introduction.html). +That's all for now! Elixir developers can see [a summary of all changes in v0.14.0 in the release notes](https://github.com/elixir-lang/elixir/releases/tag/v0.14.0). In case you are new around here, you can get started with Elixir by reading [our Getting Started guide](https://hexdocs.pm/elixir/introduction.html). We hope to see you all this July at [ElixirConf](http://elixirconf.com/)! diff --git a/_posts/2014-08-07-elixir-v0-15-0-released.markdown b/_posts/2014-08-07-elixir-v0-15-0-released.markdown index dee1c607e..32b579555 100644 --- a/_posts/2014-08-07-elixir-v0-15-0-released.markdown +++ b/_posts/2014-08-07-elixir-v0-15-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.15.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: "Elixir v0.15.0 introduces Elixir's Logger, Mix aliases and is the last stop before Elixir v1.0. We are also glad to welcome Alexei into our team!" --- @@ -162,4 +163,4 @@ Alexei is also interested in how we can extend our tooling to the Erlang ecosyst We are very close to launch Elixir v1.0! All planned features are already in Elixir's codebase and at the moment there are only [four open issues in our tracker tagged with the v1.0 milestone](https://github.com/elixir-lang/elixir/issues?q=is%3Aopen+is%3Aissue+milestone%3Av1.0). -Our estimated date for the first release candidate for Elixir v1.0 is August 30th. This means there is no better time to learn Elixir than now! If you haven't started yet, you can get started with Elixir by reading [our Getting Started guide](/getting-started/introduction.html) or by checking one of the many "Learning Resources" on the sidebar. +Our estimated date for the first release candidate for Elixir v1.0 is August 30th. This means there is no better time to learn Elixir than now! If you haven't started yet, you can get started with Elixir by reading [our Getting Started guide](https://hexdocs.pm/elixir/introduction.html) or by checking one of the many "Learning Resources" on the sidebar. diff --git a/_posts/2014-09-18-elixir-v1-0-0-released.markdown b/_posts/2014-09-18-elixir-v1-0-0-released.markdown index c1499fa2b..72929864c 100644 --- a/_posts/2014-09-18-elixir-v1-0-0-released.markdown +++ b/_posts/2014-09-18-elixir-v1-0-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v1.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v1.0 is finally out! --- @@ -53,7 +54,7 @@ These expectations also apply to future releases under the v1 branch, except for ## Learn more -You can get started with Elixir via our [Getting Started guide](/getting-started/introduction.html). There are quite some Elixir books out there too, now getting sent to the presses, quite a few can be found in the sidebar, which also includes screencasts and other resources. +You can get started with Elixir via our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html). There are quite some Elixir books out there too, now getting sent to the presses, quite a few can be found in the sidebar, which also includes screencasts and other resources. You can also learn more about Elixir by checking out [the videos from ElixirConf 2014](http://www.confreaks.com/events/elixirconf2014), the first (and so far the best) Elixir conference ever! You can learn more about [the language history](http://www.confreaks.com/videos/4134-elixirconf2014-keynote-elixir), [how Elixir can change the way you code](http://www.confreaks.com/videos/4119-elixirconf2014-opening-keynote-think-different) or [even hear stories of how Elixir is being used in production](http://www.confreaks.com/videos/4131-elixirconf2014-otp-in-production-the-nitty-gritty-details-of-game-servers). diff --git a/_posts/2015-09-28-elixir-v1-1-0-released.markdown b/_posts/2015-09-28-elixir-v1-1-0-released.markdown index e5f85c7a5..5adecea87 100644 --- a/_posts/2015-09-28-elixir-v1-1-0-released.markdown +++ b/_posts/2015-09-28-elixir-v1-1-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v1.1 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v1.1 brings enhancements, bug fixes, performance improvements and more into Elixir. --- diff --git a/_posts/2016-01-03-elixir-v1-2-0-released.markdown b/_posts/2016-01-03-elixir-v1-2-0-released.markdown index 0019743ab..1b8a7efd8 100644 --- a/_posts/2016-01-03-elixir-v1-2-0-released.markdown +++ b/_posts/2016-01-03-elixir-v1-2-0-released.markdown @@ -1,14 +1,15 @@ --- layout: post title: Elixir v1.2 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v1.2 brings enhancements, bug fixes, performance improvements and more into Elixir. --- v1.2 brings enhancements, bug fixes, performance improvements and more into Elixir. Elixir v1.2 relies on many features in Erlang 18, requiring at least Erlang 18+. Upgrading to Erlang 18 is therefore necessary before upgrading Elixir. -To celebrate this release and the new year, we have also reviewed both the [Getting Started](http://elixir-lang.org/getting-started/introduction.html) and [Mix & OTP](http://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) guides, bringing it up to date and exploring new functionalities added since Elixir v1.0. +To celebrate this release and the new year, we have also reviewed both the [Getting Started](https://hexdocs.pm/elixir/introduction.html) and [Mix & OTP](https://hexdocs.pm/elixir/introduction-to-mix.html) guides, bringing it up to date and exploring new functionalities added since Elixir v1.0. ## Erlang 18 support @@ -38,7 +39,7 @@ alias MyApp.{Foo, Bar, Baz} We have also added support for variables in map keys. Now you can write: -```iex +```elixir iex> key = :hello iex> value = "world" iex> %{key => value} @@ -112,6 +113,6 @@ These are great additions on top of the faster compilation times we have achieve With Rebar 3 gaining more adoption in the Erlang community, Mix is now able to fetch and compile Rebar 3 dependencies. This feature is currently experimental and therefore opt-in: if you have a Rebar 3 dependency, you can ask Mix to use Rebar 3 to compile it by passing the `manager: :rebar3` option. Once configured, Mix will prompt you to install Rebar 3 if it is not yet available. -The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.2.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](http://elixir-lang.org/getting-started/introduction.html) to learn more. +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.2.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. Happy coding! diff --git a/_posts/2016-06-21-elixir-v1-3-0-released.markdown b/_posts/2016-06-21-elixir-v1-3-0-released.markdown index 6d4437747..3ee45d0e0 100644 --- a/_posts/2016-06-21-elixir-v1-3-0-released.markdown +++ b/_posts/2016-06-21-elixir-v1-3-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v1.3 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v1.3 brings many improvements to the language, the compiler and its tooling, specially Mix (Elixir's build tool) and ExUnit (Elixir's test framework). --- @@ -71,7 +72,7 @@ Elixir v1.3 also introduces 3 new sigils related to the types above: This release introduces new accessors to make it simpler for developers to traverse nested data structures, traversing and updating data in different ways. For instance, given a user with a list of languages, here is how to deeply traverse the map and convert all language names to uppercase: -```iex +```elixir iex> user = %{name: "john", ...> languages: [%{name: "elixir", type: :functional}, ...> %{name: "c", type: :procedural}]} @@ -282,6 +283,6 @@ By restricting hierarchies in favor of named setups, it is straight-forward for ## Summing up -The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.3.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](http://elixir-lang.org/getting-started/introduction.html) to learn more. +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.3.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. Happy coding! diff --git a/_posts/2016-07-14-announcing-genstage.markdown b/_posts/2016-07-14-announcing-genstage.markdown index 5636fb1f9..7a853d043 100644 --- a/_posts/2016-07-14-announcing-genstage.markdown +++ b/_posts/2016-07-14-announcing-genstage.markdown @@ -1,7 +1,8 @@ --- layout: post title: Announcing GenStage -author: José Valim +authors: +- José Valim category: Announcements excerpt: GenStage is a new Elixir behaviour for exchanging events with back-pressure between Elixir processes. In this blog post we will cover the background that led us to GenStage, some example use cases, and what we are exploring for future releases. --- @@ -45,7 +46,7 @@ File.stream!("path/to/some/file") |> Enum.to_list() ``` -By using `File.stream!` and `Stream.flat_map`, we build a lazy computation that will emit a single line, break that line into words, and emit such words one by one without building huge lists in memory when enumerated. The functions in the [Stream module](https://hexdocs.pm/elixir/Stream.html) just express the computation we want to perform. The computation itself, like traversing the file or breaking into words in `flat_map`, only happens when we call a function in the `Enum` module. We have covered [the foundation for Enum and Streams](http://blog.plataformatec.com.br/2015/05/introducing-reducees/) in another article. +By using `File.stream!` and `Stream.flat_map`, we build a lazy computation that will emit a single line, break that line into words, and emit such words one by one without building huge lists in memory when enumerated. The functions in the [Stream module](https://hexdocs.pm/elixir/Stream.html) just express the computation we want to perform. The computation itself, like traversing the file or breaking into words in `flat_map`, only happens when we call a function in the `Enum` module. We have covered [the foundation for Enum and Streams](https://dashbit.co/blog/introducing-reducees) in another article. The solution above allows us to work with large datasets without loading them all into memory. For large files, it is going to provide much better performance than the eager version. However, the solution above still does not leverage concurrency. For a machine with more than one core, which is the huge majority of machines we have available today, it is a suboptimal solution. @@ -200,7 +201,7 @@ One of the use cases for GenStage is to consume data from third-party systems. T During the Elixir London Meetup, I have live-coded a short example that shows how to use `GenStage` to concurrently process data stored in a PostgreSQL database as a queue: - + ### GenStage for event dispatching @@ -350,7 +351,7 @@ File.stream!("path/to/some/file", read_ahead: 100_000) # NEW! |> Enum.to_list() ``` -Flow will look at the computations we want to perform and start a series of stages to execute our code while keeping the amount of data being transfered between processes to a minimum. If you are interested in `GenStage.Flow` and how the computations above are spread across multiple stages, [we have written some documentation based on the prototypes we have built so far](https://hexdocs.pm/gen_stage/Experimental.Flow.html). The code itself is coming in future GenStage releases. We will also have to consider how the `GenStage.Flow` API mirrors the functions in `Enum` and `Stream` to make the path from eager to concurrent clearer. +Flow will look at the computations we want to perform and start a series of stages to execute our code while keeping the amount of data being transferred between processes to a minimum. If you are interested in `GenStage.Flow` and how the computations above are spread across multiple stages, [we have written some documentation based on the prototypes we have built so far](https://hexdocs.pm/gen_stage/Experimental.Flow.html). The code itself is coming in future GenStage releases. We will also have to consider how the `GenStage.Flow` API mirrors the functions in `Enum` and `Stream` to make the path from eager to concurrent clearer. For the word counting problem with a fixed data, early experiments show a linear increase in performance with a fixed overhead of 20%. In other words, a dataset that takes 60s with a single core, takes 36s on a machine with 2 cores and 18s in one with 4 cores. All of those gains by simply moving your computations from streams to Flow. We plan to benchmark on machines with over 40 cores soon. diff --git a/_posts/2017-01-05-elixir-v1-4-0-released.markdown b/_posts/2017-01-05-elixir-v1-4-0-released.markdown new file mode 100644 index 000000000..f110bb07e --- /dev/null +++ b/_posts/2017-01-05-elixir-v1-4-0-released.markdown @@ -0,0 +1,135 @@ +--- +layout: post +title: Elixir v1.4 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.4 brings many improvements to the language, its standard library and the Mix build tool. +--- + +Elixir v1.4 brings new features, enhancements and bug fixes. The most notable changes are the addition of the `Registry` module, the `Task.async_stream/3` and `Task.async_stream/5` function which aid developers in writing concurrent software, and the new application inference and commands added to Mix. + +In this post we will cover the main additions. The complete [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.4.0) are also available. + +## Registry + +The [`Registry`](https://hexdocs.pm/elixir/Registry.html) is a new module in Elixir's standard library that allows Elixir developers to implement patterns such as name lookups, code dispatching or even a pubsub system in a simple and scalable way. + +Broadly speaking, the Registry is a local, decentralized and scalable key-value process storage. Let's break this in parts: + + * Local because keys and values are only accessible to the current node (opposite to distributed) + * Decentralized because there is no single entity responsible for managing the registry + * Scalable because performance scales linearly with the addition of more cores upon partitioning + +A registry may have unique or duplicate keys. Every key-value pair is associated to the process registering the key. Keys are automatically removed once the owner process terminates. Starting, registering and looking up keys is quite straight-forward: + +```elixir +iex> Registry.start_link(:unique, MyRegistry) +iex> {:ok, _} = Registry.register(MyRegistry, "hello", 1) +iex> Registry.lookup(MyRegistry, "hello") +[{self(), 1}] +``` + +Finally, huge thanks to [Bram Verburg](https://twitter.com/voltonez) who has performed [extensive benchmarks](https://docs.google.com/spreadsheets/d/1MByRZJMCnZ1wPiLhBEnSRRSuy1QXp8kr27PIOXO3qqg/edit#gid=0) on the registry to show it scales linearly with the number of cores by increasing the number of partitions. + +## Syntax coloring + +Elixir v1.4 introduces the ability to syntax color inspected data structures and IEx automatically relies on this feature to provide syntax coloring for evaluated shell results: + +![IEx coloring](/images/contents/iex-coloring.png) + +This behaviour can be configured via the `:syntax_colors` coloring option: + +```elixir +IEx.configure [colors: [syntax_colors: [atom: :cyan, string: :green]]] +``` + +To disable coloring altogether, simply pass an empty list to `:syntax_colors`. + +## Task.async_stream + +When there is a need to traverse a collection of items concurrently, Elixir developers often resort to tasks: + +```elixir +collection +|> Enum.map(&Task.async(SomeMod, :function, [&1])) +|> Enum.map(&Task.await/1) +``` + +The snippet above will spawn a new task by invoking `SomeMod.function(element)` for every element in the collection and then await for the task results. + +However, the snippet above will spawn and run concurrently as many tasks as there are items in the collection. While this may be fine in many occasions, including small collections, sometimes it is necessary to restrict amount of tasks running concurrently, specially when shared resources are involved. + +Elixir v1.4 adds `Task.async_stream/3` and `Task.async_stream/5` which brings some of the lessons we learned from [the GenStage project](/blog/2016/07/14/announcing-genstage/) directly into Elixir: + +```elixir +collection +|> Task.async_stream(SomeMod, :function, [], max_concurrency: 8) +|> Enum.to_list() +``` + +The code above will also start the same `SomeMod.function(element)` task for every element in the collection except it will also guarantee we have at most 8 tasks being processed at the same time. You can use `System.schedulers_online` to retrieve the number of cores and balance the processing based on the amount of cores available. + +The `Task.async_stream` functions are also lazy, allowing developers to partially consume the stream until a condition is reached. Furthermore, `Task.Supervisor.async_stream/4` and `Task.Supervisor.async_stream/6` can be used to ensure the concurrent tasks are spawned under a given supervisor. + +## Application inference + +In previous Mix versions, most of your dependencies had to be added both to your dependencies list and applications list. Here is how a `mix.exs` would look like: + +```elixir +def application do + [applications: [:logger, :plug, :postgrex]] +end + +def deps do + [{:plug, "~> 1.2"}, + {:postgrex, "~> 1.0"}] +end +``` + +This was a common source of confusion and quite error prone as many developers would not list their dependencies in the applications list. + +Mix v1.4 now automatically infers your applications list as long as you leave the `:applications` key empty. The `mix.exs` above can be rewritten to: + +```elixir +def application do + [extra_applications: [:logger]] +end + +def deps do + [{:plug, "~> 1.2"}, + {:postgrex, "~> 1.0"}] +end +``` + +With the above, Mix will automatically build your application list based on your dependencies. Developers now only need to specify which applications shipped as part of Erlang or Elixir that they require, such as `:logger`. + +Finally, if there is a dependency you don't want to include in the application runtime list, you can do so by specifying the `runtime: false` option: + + {:distillery, "> 0.0.0", runtime: false} + +We hope this feature provides a more streamlined workflow for developers who are building releases for their Elixir projects. + +## Mix install from SCM + +Mix v1.4 can now install escripts and archives from both Git and Hex, providing you with even more options for distributing Elixir code. + +This makes it possible to distribute CLI applications written in Elixir by publishing a package which builds an escript to Hex. [`ex_doc`](https://hex.pm/packages/ex_doc) has been updated to serve as an example of how to use this new functionality. + +Simply running: + + mix escript.install hex ex_doc + +will fetch `ex_doc` and its dependencies, build them, and then install `ex_doc` to `~/.mix/escripts` (by default). After adding `~/.mix/escripts` to your `PATH`, running `ex_doc` is as simple as: + + ex_doc + +You can now also install archives from Hex in this way. Since they are fetched and built on the user's machine, they do not have the same limitations as pre-built archives. However, keep in mind archives are loaded on every Mix command and may conflict with modules or dependencies in your projects. For this reason, escripts is the preferred format for sharing executables. + +It is also possible to install escripts and archives by providing a Git/GitHub repo. See `mix help escript.install` and `mix help archive.install` for more details. + +## Summing up + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.4.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Happy coding! diff --git a/_posts/2017-07-25-elixir-v1-5-0-released.markdown b/_posts/2017-07-25-elixir-v1-5-0-released.markdown new file mode 100644 index 000000000..c4f38f788 --- /dev/null +++ b/_posts/2017-07-25-elixir-v1-5-0-released.markdown @@ -0,0 +1,179 @@ +--- +layout: post +title: Elixir v1.5 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.5 integrates with Erlang/OTP 20 and bring changes that improves the language reach and the developer experience +--- + +Elixir v1.5 includes new features, enhancements, and bug fixes. While [Elixir v1.4](/blog/2017/01/05/elixir-v1-4-0-released/) focused on tools for concurrency and scalability, Elixir v1.5 brings many improvements to the developer experience and quality of life. As we will see, many of those are powered by the latest Erlang/OTP 20. This is also the last Elixir release that supports Erlang/OTP 18. + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, noscript tags with the proper links will be shown. + +## UTF-8 atoms, function names and variables + +Elixir v1.5 supports non-quoted atoms and variables to be in UTF-8 when using Erlang/OTP 20. For example: + +```elixir +test "こんにちは世界" do + assert :こんにちは世界 +end +``` + +Or: + +```elixir +saudação = "Bom dia!" +``` + +Elixir follows the recommendations in [Unicode Annex #31](http://unicode.org/reports/tr31/) to make Elixir more accessible to other languages and communities. Identifiers must still be a sequence of letters, followed by digits and combining marks. This means symbols, such as mathematical notations and emoji, are not allowed in identifiers. + +For a complete reference on Elixir syntax, see the [Syntax Reference](https://hexdocs.pm/elixir/1.5/syntax-reference.html). For technical details on Unicode support, see [Unicode Syntax](https://hexdocs.pm/elixir/1.5/unicode-syntax.html). + +## IEx helpers and breakpoints + +IEx got many enhancements to the developer experience. + +First of all, the autocompletion system is now capable of autocompleting variables and user imports: + + + +IEx also got new functions, such as `exports/1`, for listing all functions and macros in a module, and the new `runtime_info/0`: + + + +Finally, IEx also features a breakpoint system for code debugging when running on Erlang/OTP 20. The following functions have been added to aid debugging: + + * `break!/2` - sets up a breakpoint for a given `Mod.fun/arity` + * `break!/4` - sets up a breakpoint for the given module, function, arity + * `breaks/0` - prints all breakpoints and their ids + * `continue/0` - continues until the next breakpoint in the same process + * `open/0` - opens editor on the current breakpoint + * `remove_breaks/0` - removes all breakpoints in all modules + * `remove_breaks/1` - removes all breakpoints in a given module + * `reset_break/1` - sets the number of stops on the given id to zero + * `reset_break/3` - sets the number of stops on the given module, function, arity to zero + * `respawn/0` - starts a new shell (breakpoints will ask for permission once more) + * `whereami/1` - shows the current location + +Let's see an example: + + + +In the snippet above we set a breakpoint in the `URI.decode_query/2` function, which is then triggered when invoked the function. We used `whereami/1` to get more information about the surrounded code and we were also able to access the variables at place of debugging. From there, we can either set more breakpoints, remove existing breakpoints and continue execution. The session ended by calling `open`, which will open your editor at the file and line under debugging. `open/1` can also be invoked by passing any module or function, and IEx will open your editor at that place. + +The debugging functions improve the experience both within IEx and during testing. For example, if you are debugging a Phoenix application, you can start `IEx` while running your test suite with `iex -S mix test --trace` and then call `IEx.break!(MyAppWeb.UserController.index/2)` to debug the `index` action of the `UserController`. Note we gave the `--trace` flag to `mix test`, which ensures only one test runs at a time and removes any timeouts from the suite. + +## Exception.blame + +`Exception.blame/3` is a new function in Elixir that is capable of attaching debug information to certain exceptions. Currently this is used to augment `FunctionClauseError`s with a summary of all clauses and which parts of clause match and which ones didn't. Let's try it out: + + + +In the example above, an argument that did not match or guard that did not evaluate to true are shown between in red. If the terminal does not support ANSI coloring, they are wrapped in `-` instead of shown in red. + +Since blaming an exception can be expensive, `Exception.blame/3` must be used exclusively in debugging situations. It is not advised to apply it to production components such as a Logger. This feature has been integrated into the compiler, the command line, ExUnit and IEx. + +This feature also requires Erlang/OTP 20. + +## Streamlined child specs + +Elixir v1.5 streamlines how supervisors are defined and used in Elixir. Elixir now allows child specifications, which specify how a child process is supervised, to be defined in modules. In previous versions, a project using Phoenix would write: + +```elixir +import Supervisor.Spec + +children = [ + supervisor(MyApp.Repo, []), + supervisor(MyApp.Endpoint, []) +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +In Elixir v1.5, one might do: + +```elixir +children = [ + MyApp.Repo, + MyApp.Endpoint +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +The above works by calling the `child_spec/1` function on the given modules. + +This new approach allows `MyApp.Repo` and `MyApp.Endpoint` to control how they run under a supervisor. This reduces the chances of mistakes being made, such as starting an Ecto repository as a worker or forgetting to declare that tasks are temporary in a supervision tree. + +If it is necessary to configure any of the children, such can be done by passing a tuple instead of an atom: + +```elixir +children = [ + {MyApp.Repo, url: "ecto://localhost:4567/my_dev"}, + MyApp.Endpoint +] +``` + +The modules `Agent`, `Registry`, `Task`, and `Task.Supervisor` have been updated to include a `child_spec/1` function, allowing them to be used directly in a supervision tree similar to the examples above. `use Agent`, `use GenServer`, `use Supervisor`, and `use Task` have also been updated to automatically define an overridable `child_spec/1` function. + +Finally, child specifications are now provided as maps (data-structures) instead of the previous `Supervisor.Spec.worker/3` and `Supervisor.Spec.supervisor/3` APIs. This behaviour also aligns with how supervisors are configured in Erlang/OTP 18+. See the updated [`Supervisor`](https://hexdocs.pm/elixir/1.5/Supervisor.html) docs for more information, as well as the new `Supervisor.init/2` and `Supervisor.child_spec/2` functions. + +## @impl + +This release also allows developers to mark which functions in a given module are an implementation of a callback. For example, when using the [Plug](https://github.com/elixir-lang/plug) project, one needs to implement both `init/1` and `call/2` when writing a Plug: + +```elixir +defmodule MyApp do + @behaviour Plug + + def init(_opts) do + opts + end + + def call(conn, _opts) do + Plug.Conn.send_resp(conn, 200, "hello world") + end +end +``` + +The problem with the approach above is that, once more and more functions are added to the `MyApp` module, it becomes increasingly harder to know the purposes of the `init/1` and `call/2` functions. For example, for a developer unfamiliar with Plug, are those functions part of the `MyApp` API or are they implementations of a given callback? + +Elixir v1.5 introduces the `@impl` attribute, which allows us to mark that certain functions are implementation of callbacks: + +```elixir +defmodule MyApp do + @behaviour Plug + + @impl true + def init(_opts) do + opts + end + + @impl true + def call(conn, _opts) do + Plug.Conn.send_resp(conn, 200, "hello world") + end +end +``` + +You may even use `@impl Plug` if you want to explicitly document which behaviour defines the callback you are implementing. + +Overall, using `@impl` has the following advantages: + + * Readability of the code is increased, as it is now clear which functions are part of your API and which ones are callback implementations. To reinforce this idea, `@impl true` automatically marks the function as `@doc false`, disabling documentation unless `@doc` is explicitly set + + * If you define `@impl` before a function that is not a callback, Elixir will error. This is useful in case of typos or in case the behaviour definition changes (such as a new major version of a library you depend on is released) + + * If you use `@impl` in one implementation, Elixir will force you to declare `@impl` for all other implementations in the same module, keeping your modules consistent + +## Calendar improvements + +[Elixir v1.3](https://elixir-lang.org/blog/2016/06/21/elixir-v1-3-0-released/) introduced the Calendar module with the underlying `Time`, `Date`, `NaiveDateTime` and `Datetime` data types. We are glad to announce we consider the base Calendar API to be finished in Elixir v1.5. This release includes many enhancements, such as `Date.range/2` and the ability to convert between different calendars. + +## Summing up + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.5.0). There are many other exciting changes, such as compiler enhancements that reduces compilation times by 10%-15% on averages. When taken into account with the compiler improvements in Erlang/OTP 20 itself, some applications have seen gains up to 30% in compilation times. + +Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. diff --git a/_posts/2017-10-31-stream-data-property-based-testing-and-data-generation-for-elixir.markdown b/_posts/2017-10-31-stream-data-property-based-testing-and-data-generation-for-elixir.markdown new file mode 100644 index 000000000..242c28635 --- /dev/null +++ b/_posts/2017-10-31-stream-data-property-based-testing-and-data-generation-for-elixir.markdown @@ -0,0 +1,184 @@ +--- +layout: post +title: "StreamData: Property-based testing and data generation" +authors: +- Andrea Leopardi +category: Announcements +excerpt: We are working on data generation and property-based testing for the next versions of Elixir. +--- + +In this blog post, we'll talk about property-based testing and sample data generation. We'll cover what these are, why we want them in Elixir, and what are are plans for the future. If you want to use the features discussed here or you want to read more formal documentation, head over to [stream_data][], which is a library that currently provides both features (albeit in beta form) and which is where we are focusing our efforts. + +## Sample data generation + +The core of the [stream_data][] library is `StreamData`: this module provides all the functionalities related to generating sample data of many kinds. It includes both data generators for data types (like integers or booleans) as well as tools to combine other generators (such as `one_of(list_of_generators)`). + +Developers are not supposed to create generators from scratch, but use the provided generators and the provided combinator functions to compose them. An example of a generator is the one returned by `StreamData.integer()`: this function returns a generator that generates integers. Generators are infinite streams of terms that implement the `Enumerable` protocol. This means we can do operations such as taking terms out of a generator through functions from `Enum` and `Stream`: + +```elixir +Enum.take(StreamData.integer(), 5) +#=> [1, -1, 3, 4, 3] +``` + +`StreamData` contains some functions to modify generators. For example, you can build a generator of positive integers on top of `StreamData.integer()` and `StreamData.map/2`: + +```elixir +generator = StreamData.map(StreamData.integer(), &abs/1) +Enum.take(generator, 5) +#=> [0, 1, 3, 3, 2] +``` + +`StreamData.map/2` is encouraged over `Stream.map/2` because generators return values that can shrink, which is something property-based testing takes advantage of as we'll see later on. When treated as enumerables, generators return normal values that cannot be shrunk. + +We decided to separate data-generation from property-based testing because it's something that developers can take advantage of in situations outside of property-based testing. For example, data streams can be used to seed a database or to have randomly generated data available during regular tests. + +## Property-based testing + +We often write tests like this: + +```elixir +test "length/1 calculates the length of a list" do + assert length([]) == 0 + assert length([:one]) == 1 + assert length([1, 2, 3]) == 3 +end +``` + +This test is written using an *example-based approach*. We are writing both the input to the piece of software we are testing as well as the expected output, and the testing tool is verifying that running the software on the given input results in the expected output. This style of testing is common and useful because it lets you get up and running easily and also lets you test known corner cases in an explicit way. However, it's hard to test many cases this way and even harder to uncover *unknown* corner cases that may reveal bugs in your code. + +Property-based testing is an intuitive way to fix some of the problems mentioned above. + +```elixir +property "length/1 is always >= 0" do + check all list <- list_of(term()) do + assert length(list) >= 0 + end +end +``` + +With property-based testing, you specify a set of valid inputs (lists in the example above) for your code and verify that your code holds some property for values taken at random from the valid inputs. In the example above, the test takes many (usually around 100) values at random from the `list_of(term())` *generator* and verifies a property of `length/1`, that is, that `length/1` always returns a non-negative integer. A generator is just a `StreamData` generator, as we discussed in the previous section. + +### Shrinking + +Since we're generating lots of random inputs to test, inputs that cause failures are often complex and convoluted. Take this trivial example of a property: + +```elixir +property "list does not contain multiples of 4" do + check all list <- list_of(positive_integer()) do + refute Enum.any?(list, &(rem(&1, 4) == 0)) + end +end +``` + +When running this property, the failure might trigger for a list like this: + +```elixir +[19, 12, 6, 11, 2, 20, 10] +``` + +From this list, it's not easy to see why the test is failing (well, we know why because we wrote a doctored test that's supposed to fail). When running the property though, the failure that will be reported will look like this: + +``` +1) property list does not contain multiples of 7 (MyPropertyTest) + my_property_test.exs:6 + Failed with generated values (after 4 attempt(s)): + + list <- list_of(positive_integer()) + #=> [4] + + Expected false or nil, got true + code: refute Enum.any?(list, &(rem(&1, 4) == 0)) +``` + +This error shows the minimal generated value that triggers the failure, that is, `[4]`. The process of finding the minimal generated value that triggers a failure is called *shrinking*. All generators that come with `StreamData` generate values that "bundle" a way to shrink them so that property-based testing can use this to provide the shrinking functionality. Shrinking is a fundamental part of property-based testing as it takes out the *noise* of random-generated data to reduce the failing data to focused and easier-to-understand terms. + +### Using property-based testing in stream_data + +The core of property-based testing in stream_data is the `check all` macro. In this macro, you list a bunch of generators and filters (very similarly to how you would in `for` comprehensions) and then pass a body where you can verify that a property holds for the generated data. + +To make the `check all` macro available in your test, alongside importing all functions from `StreamData`, you can `use ExUnitProperties`: + +```elixir +defmodule MyPropertyTest do + use ExUnit.Case, async: true + use ExUnitProperties + + test "the in/2 operator works with lists" do + check all list <- list_of(term()), + list != [], + elem <- member_of(list) do + assert elem in list + end + end +end +``` + +As you can see, we can filter generated data (`list != []`) directly in the `check all` macro. We can also do simple assignments. The example above uses the `check all` macro inside a regular `test`. If you want that your properties are reported as "property" at the end of an ExUnit test run, you can use the `property` macro instead: + +```elixir +defmodule MyPropertyTest do + use ExUnit.Case, async: true + use ExUnitProperties + + property "the in/2 operator works with lists" do + check all list <- list_of(term()), + list != [], + elem <- member_of(list) do + assert elem in list + end + end +end +``` + +By doing this your properties will also be tagged with the `:property` tag, which means you will be able to do things like: + +```bash +mix test --only property +``` + +to run only properties. + +There's not much more to the mechanics of stream_data. Most of the work you will have to do revolves around finding good properties to test for your code and writing good generators for the data over which you want to test. Head over to [stream_data][]'s documentation for detailed documentation. + +### Advantages of property-based testing + +Using property-based testing has some advantages. First of all, it lets you test properties of your code over many more values than you otherwise would with example-based testing. While it's true that random data generation can't cover all the possible values that a piece of code can deal with, the confidence in your codebase can still increase over time because the property-based tests will likely generate different values on each run. Example-based testing means your test data will not change over time. + +Property-based testing however can also have a more powerful impact on the way you design software. When you start writing property-based tests, you will start thinking about what guarantees your code provides and what properties it satisfies. If you write properties before writing code, this can easily influence the way you write that code. + +### Learning resources + +Property-based testing is not something specific to Elixir. While having its roots in Haskell (check out the [original QuickCheck paper][quickcheck-paper] if you're interested), nowadays many languages have stable and usable implementations of it: Clojure has [test.check][], Python has [Hypothesis][], and many more. One of the most famous and complete tools for property-based testing exists for Erlang itself: [QuickCheck][] by Quviq is a complete commercial solution for property-based testing in Erlang of both stateless as well as stateful systems, and Quviq even provides a custom Erlang scheduler to test race conditions in your concurrent programs. + +A young but awesome book about property-based testing written by Fred Hebert is also available at [propertesting.com][]. This book is a *proper* (pun intended) guide to property-based testing and uses an Erlang library called [PropEr][]. However, the concepts and techniques perfectly apply to Elixir and stream_data as well. + +## Why include property-based testing in Elixir (and rewriting from scratch) + +The community has expressed some concern regarding two main things: why do we want to include a property-based testing tool in Elixir's standard library? And why write such a tool from scratch instead of using one of the existing Erlang or Elixir solutions? + +The answer to the first question is that we believe providing such a tool in the standard library will encourage developers to use property-based testing and ultimately improve their software and the way they write it. At the same time, we want to be able to use property-based testing to test the Elixir codebase itself (which already turned out great [in the past][quickcheck-pr]). + +The reasons for writing a new property-based testing library from scratch are best explained by José in [this ElixirForum post][elixirforum-post]: + +> * Since we want to bundle it as part of Elixir, the code should be open source with an appropriate license. +> * We wanted to add both data generation and property testing to Elixir. That's why the library is called stream_data instead of something named after property tests. The goal is to reduce the learning curve behind property testing by exposing the data generation aspect as streams, which is a known construct to most Elixir developers. We had this approach in mind for a while and the first library we saw leveraging this in practice was [@pragdave's pollution][pollution]. +> * Finally, since the core team are taking the responsibility of maintaining property testing as part of Elixir for potentially the rest of our lives, we want to have full understanding of every single line of code. This is non-negotiable as it guarantees we can continue to consistently improve the code as we move forward. +> +> We understand rolling our own implementation has its downsides, especially since it lacks maturity compared to alternatives, but we balance it by actively seeking input from knowledgeable folks and by listening to the feedback that comes from the community, which we are very thankful for. + +## Roadmap + +`stream_data` and the functionalities it includes are scheduled to be included in one of the next two Elixir releases, likely 1.6 but possibly 1.7. We have used the names `StreamData` and `ExUnitProperties` to avoid conflicts when those modules are eventually merged into Elixir. When merged, they will be renamed to the proper `Stream.Data` and `ExUnit.Properties` modules. Right now, all development is happening in the [stream_data][] repository, where we are discussing features and giving users a chance to try out the functionalities early on. We'd love for anyone to get involved in trying stream_data and we'd love feedback! + +**Update Jun/2020:** after careful consideration, the Elixir team decided to not include `StreamData` in Elixir itself, and keep it as package, as it is able to provide all of the necessary features without a need for direct integration with the language. + +[stream_data]: https://github.com/whatyouhide/stream_data +[quickcheck-paper]: http://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf +[test.check]: https://github.com/clojure/test.check +[Hypothesis]: https://github.com/HypothesisWorks/hypothesis-python +[QuickCheck]: http://quviq.com +[propertesting.com]: http://propertesting.com +[PropEr]: https://github.com/manopapad/proper +[quickcheck-pr]: https://github.com/elixir-lang/elixir/pull/5022#issuecomment-233195478 +[pollution]: https://github.com/pragdave/pollution +[elixirforum-post]: https://elixirforum.com/t/questions-about-property-testing-stream-data/9445/47 diff --git a/_posts/2018-01-17-elixir-v1-6-0-released.markdown b/_posts/2018-01-17-elixir-v1-6-0-released.markdown new file mode 100644 index 000000000..99bc8c441 --- /dev/null +++ b/_posts/2018-01-17-elixir-v1-6-0-released.markdown @@ -0,0 +1,157 @@ +--- +layout: post +title: Elixir v1.6 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.6 includes a code formatter, defguard, dynamic supervision and new module attributes that improves code quality and the developer experience +--- + +Elixir v1.6 includes new features, enhancements, and bug fixes. The main feature in this release is a code formatter. Important improvements can also be found in the standard library and in the Mix build tool. + +## Code formatter + +The big feature in Elixir v1.6 is the addition of [a code formatter](https://hexdocs.pm/elixir/Code.html#format_string!/2) and an accompanying `mix format` task that adds automatic formatting to your projects. + +The goal of the formatter is to automate the styling of codebases into a unique and consistent layout used across teams and the whole community. Code is now easier to write, as you no longer need to concern yourself with formatting rules. Code is also easier to read, as you no longer need to convert the styles of other developers in your mind. + +The formatter also helps new developers to learn the language by giving immediate feedback on code structure, and eases code reviews by allowing teams to focus on business rules and code quality rather than code style. + +To automatically format your codebase, you can run the [new `mix format` task](https://hexdocs.pm/mix/Mix.Tasks.Format.html). A `.formatter.exs` file may be added to your project root for rudimentary formatter configuration. The mix task also supports flags for CI integration. For instance, you can make your build or a Pull Request fail if the code is not formatted accordingly by using the `--check-formatted` flag. We also recommend developers to check their favorite editor and see if it already provides key bindings for `mix format`, allowing a file or a code snippet to be formatted without hassle. + +The Elixir codebase itself [has already been fully formatted](https://github.com/elixir-lang/elixir/issues/6643) and all further contributions are expected to contain formatted code. We recommend existing codebases to be formatted in steps. While the formatter will correctly handle long lines and complex expressions, refactoring the code by breaking those into variables or smaller functions as you format them will lead to overall cleaner and more readable codebases. + +## Dynamic Supervisor + +Supervisors in Elixir are responsible for starting, shutting down, and restarting child processes when things go wrong. Most of the interaction with supervisors happens through [the `Supervisor` module](https://hexdocs.pm/elixir/Supervisor.html) and it provides three main strategies: `:one_for_one`, `:rest_for_one` and `:one_for_all`. + +However, sometimes the children of a supervisor are not known upfront and are rather started dynamically. For example, if you are building a web server, you have each request being handled by a separate supervised process. Those cases were handled in the Supervisor module under a special strategy called `:simple_one_for_one`. + +Unfortunately, this special strategy changed the semantics of the supervisor in regards to initialization and shutdown. Plus some APIs expected different inputs or would be completely unavailable depending on the supervision strategy. + +Elixir v1.6 addresses this issue by introducing [a new `DynamicSupervisor` module](https://hexdocs.pm/elixir/DynamicSupervisor.html), which encapsulates the old `:simple_one_for_one` strategy and APIs in a proper module while allowing the documentation and API of the `Supervisor` module to focus on its main use cases. Having a separate `DynamicSupervisor` module also makes it simpler to add new features to the dynamic supervisor, such as the new `:max_children` option that limits the maximum number of children supervised dynamically. + +## `@deprecated` and `@since` attributes + +This release also introduces two new attributes associated with function definitions: `@deprecated` and `@since`. The former marks if a function or macro is deprecated, the latter annotates the version the API was introduced: + +```elixir +@doc "Breaks a collection into chunks" +@since "1.0.0" +@deprecated "Use chunk_every/2 instead" +def chunk(collection, chunk_size) do + chunk_every(collection, chunk_size) +end +``` + +The `mix xref` task was also updated to warn if your project calls deprecated code. So if a definition is marked as `@deprecated` and a module invokes it, a warning will be emitted during compilation. This effectively provides libraries and frameworks a mechanism to deprecate code without causing multiple warnings to be printed in runtime and without impacting performance. + +Note those attributes are not yet available to tools that generate documentation. Such functionality will be added in Elixir v1.7 once [Elixir adopts EEP-48](https://github.com/elixir-lang/elixir/issues/7198). We still recommend developers to start annotating their APIs so the information is already available when the tooling is updated. + +## `defguard` and `defguardp` + +Elixir provides the concepts of guards: expressions used alongside pattern matching to select a matching clause. Let's see an example straight from [Elixir's home page](https://elixir-lang.org): + +```elixir +def drive(%User{age: age}) when age >= 16 do + # Code that drives a car +end +``` + +`%User{age: age}` is matching on a `User` struct with an age field and `when age >= 16` is the guard. + +Since only a handful of constructs are [allowed in guards](https://hexdocs.pm/elixir/guards.html#content), if you were in a situation where you had to check the age to be more than or equal to 16 in multiple places, extracting the guard to a separate function would be [less than obvious and error prone](https://github.com/elixir-lang/elixir/issues/2469). To address those issues, [this release introduces `defguard/1` and `defguardp/1`](https://hexdocs.pm/elixir/Kernel.html#defguard/1): + +```elixir +defguard is_old_to_drive(age) when age >= 16 + +def drive(%User{age: age}) when is_old_to_drive(age) do + # Code that drives a car +end +``` + +## IEx improvements + +IEx also got its share of improvements. The new code formatter allows us to pretty print code snippets, types and specifications, improving the overall experience when exploring code through the terminal. + +The autocomplete mechanism also got smarter, being able to provide context autocompletion. For example, typing `t Enum.` and hitting TAB will autocomplete only the types in Enum (in contrast to all functions). Typing `b GenServer.` and hitting TAB will autocomplete only the behaviour callbacks. + +Finally, the breakpoint functionality added [in Elixir v1.5](https://elixir-lang.org/blog/2017/07/25/elixir-v1-5-0-released/) has been improved to support pattern matching and guards. For example, to pattern match on a function call when the first argument is the atom `:foo`, you may do: + +``` +iex> break! SomeFunction.call(:foo, _, _) +``` + +For more information, see [`IEx.break!/4`](https://hexdocs.pm/iex/IEx.html#break!/4). + +## mix xref + +[`mix xref`](https://hexdocs.pm/mix/Mix.Tasks.Xref.html) is a task added in Elixir v1.3 which provides general information about how modules and files in an application depend on each other. This release brings many improvements to `xref`, extending the reach of the analysis and helping developers digest the vast amount of data it produces. + +One of such additions is the `--include-siblings` option that can be given to all `xref` commands inside umbrella projects. For example, to find all of the callers of a given module or function of an application in an umbrella: + +``` +$ mix xref callers SomeModule --include-siblings +``` + +The `graph` command in `mix xref` now can also output general statistics about the graph. In [the hexpm project](https://github.com/hexpm/hexpm), you would get: + +``` +$ mix xref graph --format stats +Tracked files: 129 (nodes) +Compile dependencies: 256 (edges) +Structs dependencies: 46 (edges) +Runtime dependencies: 266 (edges) + +Top 10 files with most outgoing dependencies: + * test/support/factory.ex (18) + * lib/hexpm/accounts/user.ex (13) + * lib/hexpm/accounts/audit_log.ex (12) + * lib/hexpm/web/controllers/dashboard_controller.ex (12) + * lib/hexpm/repository/package.ex (12) + * lib/hexpm/repository/releases.ex (11) + * lib/hexpm/repository/release.ex (10) + * lib/hexpm/web/controllers/package_controller.ex (10) + * lib/mix/tasks/hexpm.stats.ex (9) + * lib/hexpm/repository/registry_builder.ex (9) + +Top 10 files with most incoming dependencies: + * lib/hexpm/web/web.ex (84) + * lib/hexpm/web/router.ex (29) + * lib/hexpm/web/controllers/controller_helpers.ex (29) + * lib/hexpm/web/controllers/auth_helpers.ex (28) + * lib/hexpm/web/views/view_helpers.ex (27) + * lib/hexpm/web/views/icons.ex (27) + * lib/hexpm/web/endpoint.ex (23) + * lib/hexpm/ecto/changeset.ex (22) + * lib/hexpm/accounts/user.ex (19) + * lib/hexpm/repo.ex (19) +``` + +`mix xref graph` also got the `--only-nodes` and `--label` options. The former asks Mix to only output file names (nodes) without the edges. The latter allows you to focus on certain relationships: + +``` +# To get all files that depend on lib/foo.ex +mix xref graph --sink lib/foo.ex --only-nodes + +# To get all files that depend on lib/foo.ex at compile time +mix xref graph --label compile --sink lib/foo.ex --only-nodes + +# To get all files lib/foo.ex depends on +mix xref graph --source lib/foo.ex --only-nodes + +# To limit statistics only to compile time dependencies +mix xref graph --format stats --label compile +``` + +Those improvements will help developers better understand the relationship between files and reveal potentially complex parts of their systems. + +Other improvements in Mix include [better compiler diagnostics](https://hexdocs.pm/mix/Mix.Task.Compiler.html) for editor integration, support for [the `--slowest N` flag in `mix test`](https://hexdocs.pm/mix/Mix.Tasks.Test.html) that shows the slowest tests in your suite, and a new [`mix profile.eprof` task](https://hexdocs.pm/mix/Mix.Tasks.Eprof.html) that provides time based profiling, complementing the existing [`mix profile.cprof` (count based)](https://hexdocs.pm/mix/Mix.Tasks.Profile.Cprof.html) and [`mix profile.fprof` (flame based)](https://hexdocs.pm/mix/Mix.Tasks.Profile.Fprof.html). + +## Summing up + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.6.0). There are many other exciting changes, such as compiler enhancements to better track dependencies, leading to less files being recompiled whenever there are changes in project, and overall faster compilation. + +Work on Elixir v1.7 has already started. We still welcome developers to try out the [previously announced StreamData library](https://elixir-lang.org/blog/2017/10/31/stream-data-property-based-testing-and-data-generation-for-elixir/), that aims to bring data generation and property-based testing to Elixir. The other [features scheduled for v1.7 can be found in the issues tracker](https://github.com/elixir-lang/elixir/issues). + +Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. diff --git a/_posts/2018-06-28-gsoc-projects.markdown b/_posts/2018-06-28-gsoc-projects.markdown new file mode 100644 index 000000000..168f6126e --- /dev/null +++ b/_posts/2018-06-28-gsoc-projects.markdown @@ -0,0 +1,86 @@ +--- +layout: post +title: Google Summer of Code 2018 projects +authors: +- Andrea Leopardi +category: Announcements +excerpt: Once again Elixir is participating in Google Summer of Code 2018. In this post, we'll have a look at the active projects. +--- + +Like previous years, the Elixir community is happy to participate in [Google Summer of Code][gsoc] 2018. We are currently working on four different projects. Let's have a look at them. + +## StreamData integration with typespecs + +*Student: Nikola Jichev* + +[StreamData][stream_data] is a data-generation and property-based testing library for Elixir. The goal of this GSoC project is to integrate StreamData with typespecs. + +The data-generation side of StreamData provides tools to generate random data through composable generators. For example, you could generate keyword lists like this: + +```elixir +import StreamData + +keywords_generator = list_of({atom(:alphanumeric), term()}) + +Enum.take(keywords_generator, 2) +#=> [[_: [true]], [tm: 2, h: %{}]] +``` + +In many cases, it would be useful to be able to generate such random data starting from already existing or user-defined types. For example, Elixir already provides a built-in `keyword/0` type for keyword lists defined roughly as: + +```elixir +@type keyword() :: [{atom(), any()}] +``` + +The goal of the first part of this GSoC project is to provide StreamData with the ability to create data generators from type definitions. The API is not yet defined, but in this case, it could look something like the following: + +```elixir +import StreamData + +keywords_generator = from_type(keyword/0) + +Enum.take(keywords_generator, 2) +#=> [[_: [true]], [tm: 2, h: %{}]] +``` + +In the second part of the GSoC project, the aim is to be able to property-test functions with specs automatically. + +```elixir +@spec has_key?(keyword(), atom()) :: boolean() +def has_key?(keyword, key) do + # ... +end +``` + +The first part of the project focuses on generating data from types, so we know how to generate function arguments. The missing piece is **validating** that a given term *belongs to* a given type. For example, in the snippet above, we want to be able to check if a term is a `boolean()`. Once we're able to do this, automatic spec validation will be straightforward: it will be a matter of generating random arguments for the given function, calling the function with those arguments, and asserting that the returned value belongs to the return type defined in the spec. + +This kind of property-based testing doesn't test for *correctness*. In the snippet above, `has_key?/2` could be implemented to ignore arguments always return `false` and the automatic spec validation would pass since `false` is always a boolean. However, this is a kind of **smoke testing** useful for discovering inconsistencies in the arguments and return values of functions. + +## Tensorflex: Tensorflow bindings for the Elixir programming language + +*Student: Anshuman Chhabra* + +Currently, there is a lack of machine learning tools and frameworks for Elixir. With the number of programmers learning/using machine learning only set to grow, supporting machine learning capabilities is essential for any programming language. Moreover, there are discussions on [ElixirForum][elixirforum] regarding this and recent talks given at ElixirConf that reflect the need for Elixir to provide machine learning capabilities. + +This project's goal is Tensorflex, an Elixir machine learning framework similar to [Keras for Python][keras]. Keras uses Tensorflow as a backend for doing all the machine learning. Tensorflex will use Using Native Implemented Functions (NIFs) and the Tensorflow C API as a backend to provide a low-level API. This low-level API will then be used to write a Keras-like framework in the form of a high-level API. This will allow Elixir developers to write expedient and efficient machine learning code in Elixir. + +## Dialyzer task for Elixir + +*Student: Gabriel Gatu* + +Dialyzer is a discrepancy analyzer that ships as part of Erlang/OTP. Currently, there are two projects that add Dialyzer support to Elixir applications: [dialyxir][] and [dialyzex][]. The goal of this project is to bring ideas from both projects into Elixir itself in order to make using Dialyzer in Elixir projects easier. The task we aim to add to Elixir will focus on two main features: better user experience (in particular, better error messages and formatting) and the ability to analyze projects incrementally. + +## ElixirBench + +*Student: Tallys Martins* + +ElixirBench aims to be a service to monitor performance of Elixir projects. The goal of the GSoC project is to bring ElixirBench up and have it run nightly performance monitoring of significant Elixir projects (including Elixir itself). The end goal is to have a platform that, given a project from GitHub, will monitor the performance of new releases of that project and look for performance regressions. The benchmarking process will be controlled through a configuration file that will specify the benchmark scripts to run. + +We have high hopes for this tool as we see value in it for the whole community and for core Elixir projects alike. + +[gsoc]: https://summerofcode.withgoogle.com +[stream_data]: https://github.com/whatyouhide/stream_data +[elixirforum]: https://elixirforum.com +[keras]: https://keras.io +[dialyxir]: https://github.com/jeremyjh/dialyxir +[dialyzex]: https://github.com/Comcast/dialyzex diff --git a/_posts/2018-07-25-elixir-v1-7-0-released.markdown b/_posts/2018-07-25-elixir-v1-7-0-released.markdown new file mode 100644 index 000000000..51e8f7a66 --- /dev/null +++ b/_posts/2018-07-25-elixir-v1-7-0-released.markdown @@ -0,0 +1,157 @@ +--- +layout: post +title: Elixir v1.7 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.7 includes many quality of life improvements, focusing on documentation, Logger and ExUnit, as well as a new Elixir Core team member! +--- + +A new semester has started, which means it is time for a new Elixir release! This release brings quality of life improvements to the documentation, to error handling, to logger reporting, and to ExUnit, Elixir's testing library. + +We are also glad to welcome Michał Muskała to the Elixir Core team. Prior to joining the team, he was [a member of the Ecto team](https://github.com/elixir-ecto/ecto), he has made [plenty of contributions to Elixir](https://github.com/elixir-lang/elixir/pulls?utf8=%E2%9C%93&q=is%3Apr+author%3Amichalmuskala), often to improve performance, and [is a frequent to contribute to Erlang/OTP too](https://github.com/erlang/otp/pulls?utf8=%E2%9C%93&q=is%3Apr+author%3Amichalmuskala)! + +## Documentation metadata + +Elixir v1.7 implements [EEP 48](http://www.erlang.org/eep/eeps/eep-0048.html). EEP 48 aims to bring documentation interoperability across all languages running on the Erlang VM. + +Furthermore, EEP 48 introduces the ability to annotate documentation with metadata, which we have made possible to leverage from Elixir: + +```elixir +@moduledoc "A brand new module" +@moduledoc authors: ["Jane", "Mary"], since: "1.4.0" +``` + +Metadata can be given to `@moduledoc`, `@typedoc` and `@doc`. + +We have updated [the ExDoc tool](https://github.com/elixir-lang/ex_doc) to start leveraging metadata in order to provide better documentation for developers. Some of the improvements include: + + * Deprecated modules, functions, callbacks and types have a warning automatically attached to them. [See the deprecated `Behaviour` module as an example](https://hexdocs.pm/elixir/Behaviour.html) + + * Functions, macros, callbacks and types now include the version in which they were added. For example, [see the top right corner of the `defguard` docs](https://hexdocs.pm/elixir/Kernel.html#defguard/1) + + * Future Elixir versions will [include its own section for guards in the documentation and in the sidebar](https://hexdocs.pm/elixir/main/Kernel.html#guards). We are currently exploring ways to [generalize this feature in ExDoc itself](https://github.com/elixir-lang/ex_doc/issues/876) + +Elixir's interactive shell, IEx, has also been updated to print metadata: + +![IEx metadata](/images/contents/iex-metadata.png) + +While Elixir allows any metadata to be given, those tools currently exhibit only `:deprecated` and `:since`. Other keys may be shown in the future. + +Those improvements are not exclusive to the standard library, they are available to every Elixir library and application. We hope one day they will be available to all applications running on the Erlang VM too. + +To access the new documentation format, developers should use [`Code.fetch_docs/1`](https://hexdocs.pm/elixir/Code.html#fetch_docs/1). We have always been proud of treating documentation as a first-class citizen and the ability to add structured information to the documentation is a further step in this direction. + +## The `__STACKTRACE__` construct + +Erlang/OTP 21.0 introduces a new way to retrieve the stacktrace that is lexically scoped and no longer relies on side-effects like `System.stacktrace/0` does. Before one would write: + +```elixir +try do + ... something that may fail ... +rescue + exception -> + log(exception, System.stacktrace()) + reraise(exception, System.stacktrace()) +end +``` + +In Elixir v1.7, this can be written as: + +```elixir +try do + ... something that may fail ... +rescue + exception -> + log(exception, __STACKTRACE__) + reraise(exception, __STACKTRACE__) +end +``` + +This change may also yield performance improvements in the future, since the lexical scope allows us to track precisely when a stacktrace is used and we no longer need to keep references to stacktrace entries after the `try` construct finishes. + +Other parts of the exception system have also been improved. For example, more information is provided in certain occurrences of `ArgumentError`, `ArithmeticError` and `KeyError` messages. + +## Erlang/OTP logger integration + +Erlang/OTP 21 includes a new `:logger` module. Elixir v1.7 fully integrates with the new `:logger` and leverages its metadata system. The `Logger.Translator` mechanism has also been improved to export metadata, allowing custom Logger backends to leverage information such as: + + * `:crash_reason` - a two-element tuple with the throw/error/exit reason as the first argument and the stacktrace as the second + + * `:initial_call` - the initial call that started the process + + * `:registered_name` - the process' registered name as an atom + +We recommend Elixir libraries that previously hooked into Erlang's `:error_logger` to hook into `Logger` instead, in order to support all current and future Erlang/OTP versions. + +## Logger compile-time purging + +Previously, Logger macros such as `debug`, `info`, and so on would always evaluate their arguments, even when nothing would be logged. From Elixir v1.7 the arguments are only evaluated when the message is logged. + +The Logger configuration system also accepts a new option called `:compile_time_purge_matching` that allows you to remove log calls with specific compile-time metadata. For example, to remove all logger calls from application `:foo` with level lower than `:info`, as well as remove all logger calls from `Bar.foo/3`, you can use the following configuration: + +```elixir +config :logger, + compile_time_purge_matching: [ + [application: :foo, level_lower_than: :info], + [module: Bar, function: "foo/3"] + ] +``` + +## ExUnit improvements + +[ExUnit](https://hexdocs.pm/ex_unit/) is Elixir's unit testing library. ExUnit has always leveraged Elixir macros to provide excellent error reports when a failure happens. For example, the following code: + +```elixir +assert "fox jumps over the lazy dog" == "brown fox jumps over the dog" +``` + +will fail with the following report: + +![ExUnit Diff](/images/contents/exunit-diff.png) + +The `assert` macro is able to look at the code, extract the current file, the line, extract the operands and show a diff between the data structures alongside the stacktrace when the assertion fails. + +However, sometimes we need to write assertions such as `assert some_function(expr1, var2)`. When such assertion fails, we usually have to re-run the tests, now debugging or printing the values of `expr1` and `var2`. In Elixir v1.7, when a "bare" assertion fails, we will print the value of each argument individually. For a simple example such as `assert some_vars(1 + 2, 3 + 4)`, we get this report: + +![ExUnit Bare Assertion Diff](/images/contents/exunit-bare-assertion-diff.png) + +We have also [added coloring and diffing to doctests](https://hexdocs.pm/ex_unit/ExUnit.DocTest.html#content). + +While ExUnit is our test framework, Mix is our build tool. Developers typically run their tests by calling `mix test`. + +On the `mix test` side of things, there is a new `--failed` flag that runs all tests that failed the last time they ran. Finally, coverage reports generated with `mix test --cover` include a summary out of the box: + +``` +Generating cover results ... + +Percentage | Module +-----------|-------------------------- + 100.00% | Plug.Exception.Any + 100.00% | Plug.Adapters.Cowboy2.Stream + 100.00% | Collectable.Plug.Conn + 100.00% | Plug.Crypto.KeyGenerator + 100.00% | Plug.Parsers + 100.00% | Plug.Head + 100.00% | Plug.Router.Utils + 100.00% | Plug.RequestId + ... | ... +-----------|-------------------------- + 77.19% | Total +``` + +## Summing up + +We are really proud of this release, as it focuses mostly on quality of life improvements, instead of flashy new features. As Elixir continues to mature, expect more releases like this one. The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.7.0). + +We have also seen important developments in other areas not directly related to the Elixir codebase: + + * We have added [a "Development" section](https://elixir-lang.org/development.html) to the website, that outlines the Elixir team structure and goals + + * [Elixir now has its own mini-documentary](https://www.youtube.com/watch?v=lxYFOM3UJzo) by Honeypot + + * We have already highlighted some of the improvements in the ExDoc tool. Another improvement worth mentioning is the syntax highlighting is now done in Elixir itself, via the [Makeup](https://github.com/tmbb/makeup) library. This gives us more control over the grammar, the style, and improves load times. If you would like to add support for other languages, [reach out](https://github.com/tmbb/makeup)! + +Finally, don't forget [ElixirConf US](https://elixirconf.com/) is coming soon, in Bellevue, WA, September 4-7. Last year my keynote focused on the last 5 years with Elixir. This year we are ready to look into the 5 years ahead. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. diff --git a/_posts/2019-01-14-elixir-v1-8-0-released.markdown b/_posts/2019-01-14-elixir-v1-8-0-released.markdown new file mode 100644 index 000000000..6da84a7dc --- /dev/null +++ b/_posts/2019-01-14-elixir-v1-8-0-released.markdown @@ -0,0 +1,83 @@ +--- +layout: post +title: Elixir v1.8 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.8 comes with many improvements at the infrastructure level, improving compilation time, speeding up common patterns, and adding features around introspection of the system. +--- + +Elixir v1.8 comes with many improvements at the infrastructure level, improving compilation time, speeding up common patterns, and adding features around introspection of the system. + +## Custom struct inspections + +Elixir now provides a derivable implementation of the `Inspect` protocol. In a nutshell, this means it is really easy to filter data from your data structures whenever they are inspected. For example, imagine you have a user struct with security and privacy sensitive information: + +```elixir +defmodule User do + defstruct [:id, :name, :age, :email, :encrypted_password] +end +``` + +By default, if you inspect a user via `inspect(user)`, it will include all fields. This can cause fields such as `:email` and `:encrypted_password` to appear in logs, error reports, etc. You could always define a custom implementation of the `Inspect` protocol for such cases but Elixir v1.8 makes it simpler by allowing you to derive the `Inspect` protocol: + +```elixir +defmodule User do + @derive {Inspect, only: [:id, :name, :age]} + defstruct [:id, :name, :age, :email, :encrypted_password] +end +``` + +Now all user structs will be printed with all remaining fields collapsed: + + #User + +You can also pass `@derive {Inspect, except: [...]}` in case you want to keep all fields by default and exclude only some. + +## Time zone database support + +In Elixir v1.3, Elixir added four types, known as Calendar types, to work with dates and times: `Time`, `Date`, `NaiveDateTime` (without time zone), and `DateTime` (with time zone). Over the last versions we have added many enhancements to the Calendar types but the `DateTime` module always evolved at a slower pace since Elixir did not provide an API for time zone databases. + +Elixir v1.8 now defines a `Calendar.TimeZoneDatabase` behaviour, allowing developers to bring in their own time zone databases. By defining an explicit contract for time zone behaviours, Elixir can now extend the `DateTime` API, adding functions such as `DateTime.shift_zone/3`. By default, Elixir ships with a time zone database called `Calendar.UTCOnlyTimeZoneDatabase` that only handles UTC. + +Other Calendar related improvements include the addition of `Date.day_of_year/1`, `Date.quarter_of_year/1`, `Date.year_of_era/1`, and `Date.day_of_era/1`. + +## Faster compilation and other performance improvements + +Due to improvements to the compiler made over the last year, Elixir v1.8 should compile code about 5% faster on average. This is yet another release where we have been able to reduce compilation times and provide a more joyful development experience to everyone. + +The compiler also emits more efficient code for range checks in guards (such as `x in y..z`), for charlists with interpolation (such as `'foo #{bar} baz'`), and when working with records via the `Record` module. + +Finally, EEx templates got their own share of optimizations, emitting more compact code that runs faster. + +## Improved instrumentation and ownership with `$callers` + +The `Task` module is one of the most common ways to spawn light-weight processes to perform work concurrently. Whenever you spawn a new process, Elixir annotates the parent of that process through the `$ancestors` key. This information can be used by instrumentation tools to track the relationship between events occurring within multiple processes. However, many times, tracking only the `$ancestors` is not enough. + +For example, we recommend developers to always start tasks under a supervisor. This provides more visibility and allows us to control how those tasks are terminated when a node shuts down. In your code, this can be done by invoking something like: `Task.Supervisor.start_child(MySupervisor, task_specification)`. This means that, although your code is the one who invokes the task, the actual parent of the task would be the supervisor, as the supervisor is the one spawning it. We would list the supervisor as one of the `$ancestors` for the task, but the relationship between your code and the task is lost. + +In Elixir v1.8, we now track the relationship between your code and the task via the `$callers` key in the process dictionary, which aligns well with the existing `$ancestors` key. Therefore, assuming the `Task.Supervisor` call above, we have: + + [your code] -- calls --> [supervisor] ---- spawns --> [task] + +which means we store the following relationships: + + [your code] [supervisor] <-- ancestor -- [task] + ^ | + |--------------------- caller ---------------------| + +When a task is spawned directly from your code, without a supervisor, then the process running your code will be listed under both `$ancestors` and `$callers`. + +This small feature is very powerful. It allows instrumentation and monitoring tools to better track and relate the events happening in your system. This feature can also be used by tools like the "Ecto Sandbox". The "Ecto Sandbox" allows developers to run tests concurrently against the database, by using transactions and an ownership mechanism where each process explicitly gets a connection assigned to it. Without `$callers`, every time you spawned a task that queries the database, the task would not know its caller, and therefore it would be unable to know which connection was assigned to it. This often meant features that rely on tasks could not be tested concurrently. With `$callers`, figuring out this relationship is trivial and you have more tests using the full power of your machine. + +## Summing up + +We are really proud of this release (as usual!) which brings many improvements at the infrastructure level. Those improvements were designed with feedback from the community and from the many different companies using Elixir in production. The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.8.0). + +There is only one last major feature planned for upcoming Elixir versions, which is the addition of `mix release` to Elixir itself, streamlining the experience provided by packages like [distillery](https://github.com/bitwalker/distillery). With `mix release`, a developer can bundle the VM and all compiled code in a single directory, which can then be packaged and sent to production. We are glad to say the [work on this feature has already started](https://github.com/elixir-lang/elixir/issues/8612). + +During [my keynote at ElixirConf 2018 US](https://www.youtube.com/watch?v=suOzNeMJXl0), I talked about the next five years for Elixir and much of the emphasis is put on the community. Elixir was designed to be an extensible language and therefore the work on the language itself is meant to reduce with time, which we have seen in the last two releases. We trust the community to continue building on this solid foundation, bringing new challenges to the ecosystem and taking the language to new domains. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Your turn. :) \ No newline at end of file diff --git a/_posts/2019-02-25-mint-a-new-http-library-for-elixir.markdown b/_posts/2019-02-25-mint-a-new-http-library-for-elixir.markdown new file mode 100644 index 000000000..3868b94e6 --- /dev/null +++ b/_posts/2019-02-25-mint-a-new-http-library-for-elixir.markdown @@ -0,0 +1,80 @@ +--- +layout: post +title: Mint, a new HTTP client for Elixir +authors: +- Eric Meadows-Jönsson +category: Announcements +excerpt: Mint is a new low-level HTTP client that aims to provide a small and functional core that others can build on top. +--- + +[Mint](https://github.com/ericmj/mint) is a new low-level HTTP client that aims to provide a small and functional core that others can build on top. Mint is connection based: each connection is a single struct with an associated socket belonging to the process that started the connection. Since no extra processes are started for the connection, you can choose the process architecture that better fits your application. + +To validate this we built out the library with a common API supporting both HTTP/1 and HTTP/2 with automatic version negotiation. In addition, Mint comes with a [CA certificate store](https://github.com/ericmj/castore) to do safe by default HTTPS connections. + +## Connections without processes + +Mint HTTP connections are managed directly in the process that starts the connection, which means no connection pool is used nor new processes spawned when a connection is opened. This allows the user of the library to build their own process structure that fits their application. + +With Mint each connection has a single immutable data structure that the user needs to manage. Mint uses ["active mode"](http://www.erlang.org/doc/man/inet.html#setopts-2) sockets. This means data and events from the socket are sent as messages to the process that started the connection. The user passes the messages to the `stream/2` function that returns the updated connection and a list of "responses". Responses are streamed back which means you won't receive a single full HTTP response back from `stream/2`, instead the response is returned in partial response chunks. A chunk can be the status line, HTTP headers, or part of the response body. + +Let's look at an example of sending a request with Mint: + +```elixir +iex(1)> {:ok, conn} = Mint.HTTP.connect(:http, "httpbin.org", 80) +iex(2)> {:ok, conn, request_ref} = Mint.HTTP.request(conn, "GET", "/", [], "") +iex(3)> receive do +...(3)> message -> +...(3)> IO.inspect(message, label: :message) +...(3)> {:ok, conn, responses} = Mint.HTTP.stream(conn, message) +...(3)> IO.inspect(responses, label: :responses) +...(3)> end +message: {:tcp, #Port<0.8>, "HTTP/1.1 200 OK\r\n" <> ...} +responses: [ + {:status, #Reference<...>, 200}, + {:headers, #Reference<...>, [{"connection", "keep-alive"}, ...}, + {:data, #Reference<...>, "" <> ...}, + {:done, #Reference<...>} +] +``` + +As we can see all calls to `Mint.HTTP` functions return an updated `conn` which holds the state for the connection. It is important to carry on the `conn` to the next function call or the state will be corrupted. + +On line 2 we send a request to the server. A reference to the request is returned: this reference is useful when sending concurrent requests, either with HTTP/1 pipelining or with HTTP/2 multiplexed streams. + +Next we start a receive block waiting for a TCP active mode message and pass it to `stream/2`. The message is parsed and the response to the request is returned. As you can see the response is split over multiple tuples: `:status`, `:headers`, `:data`, and `:done`. This is because Mint was built from the ground with streaming in mind. The parts of the response will be returned continuously as TCP messages are passed to `stream/2` so that we don't have to wait for the full response to complete before starting to process it. + +If the response body is larger than a single packet `stream/2` may return multiple `:data` tuples and if the response includes trailing headers multiple `:headers` will be returned. When the response is complete `:done` will be returned. + +Note that if you send concurrent requests on a HTTP/2 connection responses can be returned interleaved from the requests using HTTP/2's stream multiplexing. Additionally, responses can be spread over multiple messages so we may need to continually receive messages and pass them to `stream/2`. + +See more examples on how to use Mint in the [documentation](https://hexdocs.pm/mint). + +## Why process-less? + +Mint may seem more cumbersome to use than most other HTTP libraries you have used and that is true in many ways. But by providing a low-level API without a predetermined process architecture it gives more flexibility to the user of the library. + +Many times you do not need a general purpose connection pool and can avoid the additional complexity, single point of failure, and potential performance bottlenecks that it brings. For example, if you are building quick CLI scripts, you most likely don't need a pool and performing a single one-off request with Mint is good enough. + +Another good use case for Mint is [GenStage](https://github.com/elixir-lang/gen_stage). If you write GenStage pipelines, it is most likely that you have a pool of producers that fetch data from external sources via HTTP. If you are using a high-level HTTP library, that comes with its own pool, now you have two pools, one of GenStage producers and another from the HTTP library. With Mint, you can have each GenStage producer manage its own connection, reducing overhead and simplifying the code. + +Of course, none of this stops you from building a connection pool on top of Mint. The point is exactly that Mint won't impose an architecture onto you. At the end of the day, we hope Mint will be a useful building block for more complex scenario and use cases. + +## HTTP/1 and HTTP/2 + +The `Mint.HTTP` module has a single interface for both HTTP/1 and HTTP/2 connections and performs version negotiation on HTTPS connections, HTTP connections default to HTTP/1. You can specify which HTTP version you want to use or use the `Mint.HTTP1` or `Mint.HTTP2` modules directly if you want to use version-specific features. + +## Safe-by-default HTTPS + +When connecting over HTTPS, Mint will perform certificate verification by default. We believe it's crucial that an HTTP library defaults to be secure out of the box. + +Mint uses an optional dependency on [CAStore](https://github.com/ericmj/castore) to provide certificates from [Mozilla's CA Certificate Store](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/). + +You can of course tweak specific SSL settings without re-building the safe defaults yourself. + +## Current state of the library + +The first version of Mint has just been released. It is an experimental library trying a new approach to building HTTP libraries so don't expect a fully stable API yet. + +Use Mint to explore new ideas for HTTP connection management and building higher level clients on top of Mint. In the future connection pooling and a higher level API may be added to supplement the current low level API, either directly to Mint or via different libraries. + +*Note:* Mint is being announced in the official Elixir blog because it was originally being considered for inclusion in Elixir itself. However, at some point the Elixir team decided it doesn't make sense to include an HTTP client in Elixir itself, at least as long as Erlang/OTP ships with a client too. Mint is not maintained by the Elixir team, although it is maintained by Eric and Andrea, who are part of the team. diff --git a/_posts/2019-06-24-elixir-v1-9-0-released.markdown b/_posts/2019-06-24-elixir-v1-9-0-released.markdown new file mode 100644 index 000000000..b1b72fd4c --- /dev/null +++ b/_posts/2019-06-24-elixir-v1-9-0-released.markdown @@ -0,0 +1,102 @@ +--- +layout: post +title: Elixir v1.9 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.9 is out with releases support, improved configuration and more. +--- + +Elixir v1.9 is out with releases support, improved configuration, and more. + +We are also glad to announce [Fernando Tapia Rico](https://github.com/fertapric) has joined the Elixir Core Team. Fernando has been extremely helpful in keeping the issues tracker tidy, by fixing bugs and improving Elixir in many different areas, such as the code formatter, IEx, the compiler, and others. + +Now let's take a look at what's new in this new version. + +## Releases + +The main feature in Elixir v1.9 is the addition of releases. A release is a self-contained directory that consists of your application code, all of its dependencies, plus the whole Erlang Virtual Machine (VM) and runtime. Once a release is assembled, it can be packaged and deployed to a target as long as the target runs on the same operating system (OS) distribution and version as the machine running the [`mix release`](https://hexdocs.pm/mix/Mix.Tasks.Release.html) command. + +Releases have always been part of the Elixir community thanks to Paul Schoenfelder's work on [Distillery](https://github.com/bitwalker/distillery) (and EXRM before that). Distillery was announced in July 2016. Then in 2017, [DockYard](https://dockyard.com/) hired Paul to work on improving deployments, an effort that would lead to [Distillery 2.0](https://dockyard.com/blog/2018/08/23/announcing-distillery-2-0). Distillery 2.0 provided important answers in areas where the community was struggling to establish conventions and best practices, such as configuration. + +At the beginning of this year, thanks to [Plataformatec](http://plataformatec.com.br/), I was able to prioritize the work on bringing releases directly into Elixir. Paul was aware that we wanted to have releases in Elixir itself and during [ElixirConf 2018](https://elixirconf.com) I announced that releases was the last planned feature for Elixir. + +The goal of Elixir releases was to double down on the most important concepts provided by Distillery and provide extensions points for the other bits the community may find important. [Paul](http://github.com/bitwalker/) and [Tristan](https://github.com/tsloughter) (who maintains [Erlang's relx](https://github.com/erlware/relx)) provided excellent feedback on Elixir's implementation, which we are very thankful for. [The Hex package manager is already using releases in production](https://dashbit.co/blog/updating-hex-pm-to-use-elixir-releases) and we also got feedback from other companies doing the same. + +Enough background, let's see why you would want to use releases and how to assemble one. + +### Why releases? + +Releases allow developers to precompile and package all of their code and the runtime into a single unit. The benefits of releases are: + + * Code preloading. The VM has two mechanisms for loading code: interactive and embedded. By default, it runs in the interactive mode which dynamically loads modules when they are used for the first time. The first time your application calls `Enum.map/2`, the VM will find the `Enum` module and load it. There's a downside. When you start a new server in production, it may need to load many other modules, causing the first requests to have an unusual spike in response time. Releases run in embedded mode, which loads all available modules upfront, guaranteeing your system is ready to handle requests after booting. + + * Configuration and customization. Releases give developers fine grained control over system configuration and the VM flags used to start the system. + + * Self-contained. A release does not require the source code to be included in your production artifacts. All of the code is precompiled and packaged. Releases do not even require Erlang or Elixir in your servers, as they include the Erlang VM and its runtime by default. Furthermore, both Erlang and Elixir standard libraries are stripped to bring only the parts you are actually using. + + * Multiple releases. You can assemble different releases with different configuration per application or even with different applications altogether. + + * Management scripts. Releases come with scripts to start, restart, connect to the running system remotely, execute RPC calls, run as daemon, run as a Windows service, and more. + +### 1, 2, 3: released assembled! + +You can start a new project and assemble a release for it in three easy steps: + + $ mix new my_app + $ cd my_app + $ MIX_ENV=prod mix release + +A release will be assembled in `_build/prod/rel/my_app`. Inside the release, there will be a `bin/my_app` file which is the entry point to your system. It supports multiple commands, such as: + + * `bin/my_app start`, `bin/my_app start_iex`, `bin/my_app restart`, and `bin/my_app stop` - for general management of the release + + * `bin/my_app rpc COMMAND` and `bin/my_app remote` - for running commands on the running system or to connect to the running system + + * `bin/my_app eval COMMAND` - to start a fresh system that runs a single command and then shuts down + + * `bin/my_app daemon` and `bin/my_app daemon_iex` - to start the system as a daemon on Unix-like systems + + * `bin/my_app install` - to install the system as a service on Windows machines + +### Hooks and Configuration + +Releases also provide built-in hooks for configuring almost every need of the production system: + + * `config/config.exs` (and `config/prod.exs`) - provides build-time application configuration, which is executed when the release is assembled + + * `config/releases.exs` - provides runtime application configuration. It is executed every time the release boots and is further extensible via config providers + + * `rel/vm.args.eex` - a template file that is copied into every release and provides static configuration of the Erlang Virtual Machine and other runtime flags + + * `rel/env.sh.eex` and `rel/env.bat.eex` - template files that are copied into every release and executed on every command to set up environment variables, including ones specific to the VM, and the general environment + +We have written [extensive documentation on releases](https://hexdocs.pm/mix/Mix.Tasks.Release.html), so we recommend checking it out for more information. + +## Configuration + +We also use the work on releases to streamline Elixir's configuration API. A new `Config` module has been added to Elixir. The previous configuration API, `Mix.Config`, was part of the Mix build tool. However, since releases provide runtime configuration and Mix is not included in releases, we ported the `Mix.Config` API to Elixir. In other words, `use Mix.Config` has been soft-deprecated in favor of `import Config`. + +Another important change related to configuration is that `mix new` will no longer generate a `config/config.exs` file. [Relying on configuration is undesired for most libraries](https://hexdocs.pm/elixir/library-guidelines.html#avoid-application-configuration) and the generated config files pushed library authors in the wrong direction. Furthermore, `mix new --umbrella` will no longer generate a configuration for each child app, instead all configuration should be declared in the umbrella root. That's how it has always behaved, we are now making it explicit. + +## Other improvements + +There are many other enhancements in Elixir v1.9. The Elixir CLI got a handful of new options in order to best support releases. `Logger` now computes its sync/async/discard thresholds in a decentralized fashion, reducing contention. `EEx` (Embedded Elixir) templates support more complex expressions than before. Finally, there is a new `~U` sigil for working with UTC DateTimes as well as new functions in the `File`, `Registry`, and `System` modules. + +## What's next? + +As mentioned earlier, releases was the last planned feature for Elixir. We don't have any major user-facing feature in the works nor planned. I know for certain some will consider this fact the most excing part of this announcement! + +Of course, it does not mean that v1.9 is the last Elixir version. We will continue shipping new releases every 6 months with enhancements, bug fixes and improvements. You can see the [Issues Tracker](http://github.com/elixir-lang/elixir/issues) for more details. + +We also are working on some structural changes. One of them is move the `mix xref` pass straight into the compiler, which would allow us to emit undefined function and deprecation warnings in more places. We are also considering a move to [Cirrus-CI](https://cirrus-ci.org/), so we can test Elixir on Windows, Unix, and FreeBSD through a single service. + +It is also important to highlight that there are two main reasons why we can afford to have an empty backlog. + +First of all, Elixir is built on top of Erlang/OTP and we simply leverage all of the work done by Ericsson and the OTP team on the runtime and Virtual Machine. The Elixir team has always aimed to contribute back as much as possible and those contributions have increased in the last years. + +Second, Elixir was designed to be an extensible language. The same tools and abstractions we used to create and enhance the language are also available to libraries and frameworks. This means the community can continue to improve the ecosystem without a need to change the language itself, which would effectively become a bottleneck for progress. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. We have also updated our [advanced Mix & OTP](https://hexdocs.pm/elixir/introduction-to-mix.html) to talk about releases. If you are looking for a more fast paced introduction to the language, see the [How I Start: Elixir](http://howistart.org/posts/elixir/1/index.html) tutorial, which has also been brought to the latest and greatest. + +Have fun! diff --git a/_posts/2020-01-27-elixir-v1-10-0-released.markdown b/_posts/2020-01-27-elixir-v1-10-0-released.markdown new file mode 100644 index 000000000..f2eac58ce --- /dev/null +++ b/_posts/2020-01-27-elixir-v1-10-0-released.markdown @@ -0,0 +1,171 @@ +--- +layout: post +title: Elixir v1.10 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.10 is out with standard library, compiler, and releases improvements. +--- + +Elixir v1.10 comes with improvements to the standard library, the compiler, as well as several additions to the [`mix release` feature added in v1.9](/blog/2019/06/24/elixir-v1-9-0-released/). In particular, this version adds a lot of polish to existing features, such as our configuration system and our sorting APIs. + +Also note Elixir v1.10 requires Erlang/OTP 21+. This allows us to provide tighter integration with Erlang/OTP's new logger. This means that the logger level, logger metadata, as well as all log messages are now shared between Erlang and Elixir APIs. + +Let's take a look at what else is new. + +## Releases improvements + +Elixir v1.9 introduced releases as a mechanism to package self-contained applications. Elixir v1.10 further improves releases with bug fixes and new enhancements based on feedback we got from the community. The highlights are: + + * Allow the dual boot system of releases to be disabled on environments that are boot-time sensitive, such as embedded devices + + * Track and raise if compile-time configuration is set or changes at runtime (more in the next section) + + * Support overlays to easily add extra files to a packaged releases + + * Allow `RELEASE_DISTRIBUTION` to be set to `none` in order to fully disable distribution + + * Add a built-in `:tar` step that automatically packages releases + +See the [full release notes for more improvements](https://github.com/elixir-lang/elixir/releases/tag/v1.10.0). + +## Improvements to sort-based APIs in Enum + +[`Enum.sort/1`](https://hexdocs.pm/elixir/Enum.html#sort/1) in Elixir by default sorts from lowest to highest: + +```elixir +iex> Enum.sort(["banana", "apple", "pineapple"]) +["apple", "banana", "pineapple"] +``` + +If you want to sort from highest to lowest, you need to call `Enum.sort/2` with a custom sorting function, such as `Enum.sort(collection, &>=/2)`, which is not immediately obvious to someone reading the code: + +```elixir +iex> Enum.sort(["banana", "apple", "pineapple"], &>=/2) +["pineapple", "banana", "apple"] +``` + +Furthermore, comparison operators, such as `<=` and `>=`, perform structural sorting, instead of a semantic one. For example, using `>=` to sort dates descendingly won't yield the correct result: + +```elixir +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]]) +[~D[2020-01-01], ~D[2019-12-31]] +``` + +To perform proper semantic comparison for dates, one would also need to pass a custom sorting function: + +```elixir +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]], &(Date.compare(&1, &2) != :lt)) +[~D[2019-12-31], ~D[2020-01-01]] +``` + +Elixir v1.10 streamlines the sorting functions by introducing both `:asc` and `:desc` shortcuts: + +```elixir +iex> Enum.sort(["banana", "apple", "pineapple"], :asc) +["apple", "banana", "pineapple"] +iex> Enum.sort(["banana", "apple", "pineapple"], :desc) +["pineapple", "banana", "apple"] +``` + +As well as adding the possibility to pass a module to perform semantic comparisons. For example, to sort dates, one now only needs to pass the `Date` module or even `{:desc, Date}` for descending semantical sort: + +```elixir +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]], Date) +[~D[2019-12-31], ~D[2020-01-01]] +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]], {:desc, Date}) +[~D[2020-01-01], ~D[2019-12-31]] +``` + +These API improvements make the code more concise and readable and they have also been added to `Enum.sort_by`, `Enum.min_by`, `Enum.max_by`, and friends. + +### Tracking of compile-time configuration + +In Elixir, we organize our code in applications. Libraries, your dependencies, and your own project are all separate applications. All applications in Elixir also come with an application environment. + +The application environment is a key-value store that allows us to configure said application. While reading the application environment at runtime is the preferred approach, in some rare occasions you may want to use the application environment to configure the compilation of a certain project. This is often done by calling `Application.get_env/3` outside of a function: + +```elixir +defmodule MyApp.DBClient do + @db_host Application.get_env(:my_app, :db_host, "db.local") + + def start_link() do + SomeLib.DBClient.start_link(host: @db_host) + end +end +``` + +This approach has one big limitation: if you change the value of the application environment after the code is compiled, the value used at runtime is not going to change! For example, if you are using `mix release` and your `config/releases.exs` has: + + config :my_app, :db_host, "db.production" + +Because `config/releases.exs` is read after the code is compiled, the new value will have no effect as the code was compiled to connect to "db.local". + +Of course, the obvious solution to this mismatch is to not read the application environment at compilation time in the first place, and instead move the code to inside a function: + +```elixir +defmodule MyApp.DBClient do + def start_link() do + SomeLib.DBClient.start_link(host: db_host()) + end + + defp db_host() do + Application.get_env(:my_app, :db_host, "db.local") + end +end +``` + +While this is the preferred approach, there are still two scenarios we need to address: + + 1. Not everyone may be aware of this pitfall, so they will mistakenly read the application environment at compile-time, until they are bitten by this behaviour + + 2. In rare occasions, you truly need to read the application environment at compile-time, and you want to be warned when you try to configure at runtime something that is valid only at compilation time + +Elixir v1.10 aims to solve these two scenarios by introducing a `Application.compile_env/3` function. For example, to read the value at compile time, you can now do: + +```elixir +@db_host Application.compile_env(:my_app, :db_host, "db.local") +``` + +By using `compile_env/3`, Elixir will store the values used during compilation and compare them with the runtime values whenever your system starts, raising an error in case they differ. This helps developers ensure they are running their production systems with the configuration they intend to. + +In future versions, we will deprecate the use `Application.get_env/3` at compile-time with a clear message pointing users to configuration best practices, effectively addressing the scenario where users read from the application environment at compile time unaware of its pitfalls. + +### Compiler tracing + +This release brings enhancements to the Elixir compiler and adds new capabilities for developers to listen to compilation events. + +In previous Elixir versions, Elixir would compile a database of cross references between modules (such as function calls, references, structs, etc) for each project in order to perform all kinds of checks, such as deprecations and undefined functions. + +Although this database was not public, developers would still use it to run their own checks against their projects. With time, developers would request more data to be included in the database, which was problematic as Elixir itself did not have a use for the additional data, and the database was not meant to be used externally in the first place. + +In Elixir v1.10, we have addressed these problems by [introducing compiler tracing](https://hexdocs.pm/elixir/Code.html#module-compilation-tracers). The compiler tracing allows developers to listen to events as they are emitted by the compiler, so they can store all of the information they need - and only the information they need. + +Elixir itself is using the new compiler tracing to provide new functionality. One advantage of this approach is that developers can now disable undefined function warnings directly on the callsite. For example, imagine you have an optional dependency which may not be available in some cases. You can tell the compiler to skip warning on calls to optional modules with: + + @compile {:no_warn_undefined, OptionalDependency} + defdelegate my_function_call(arg), to: OptionalDependency + +Previously, this information had to be added to the overall project configuration, which was far away from where the optional call effectively happened. + +### Other enhancements + +Elixir's calendar data types got many improvements, such as sigil support for third-party calendars, as well as the additions of [`DateTime.now!/2`](https://hexdocs.pm/elixir/DateTime.html#now!/2), [`DateTime.shift_zone!/3`](https://hexdocs.pm/elixir/DateTime.html#shift_zone!/3), and [`NaiveDateTime.local_now/0`](https://hexdocs.pm/elixir/NaiveDateTime.html#local_now/0). + +There are many improvements related to Elixir's AST in this release too. [`Code.string_to_quoted/2`](https://hexdocs.pm/elixir/Code.html#string_to_quoted/2) has two new options, `:token_metadata` and `:literal_encoder`, that give more control over Elixir's parser. This information was already available to the Elixir code formatter and has now been made public. These changes alongside compiler tracing means tools like [Credo](https://github.com/rrrene/credo), [Boundary](https://github.com/sasa1977/boundary), and IDE integrations have an even better foundation to analyze the source code. + +[ExUnit](https://hexdocs.pm/ex_unit), our test framework, ships two small but important improvements: `ExUnit.CaptureIO` can now be used by tests that run concurrently and we have added "pattern-matching diffing". To understand the last feature, take this code: + +```elixir +assert %{"status" => 200, "body" => %{"key" => "foo"}} = json_payload +``` + +Now imagine that `json_payload` is a large JSON blob and the `"key"` inside the `"body"` did not have value of `"foo"`. In previous Elixir versions, if the assertion failed, Elixir would print the right side and let you up to your own devices to figure out what went wrong. In Elixir v1.10, we diff the data structure against the pattern so you can see exactly which parts of the data matched the pattern and which ones did not. Note ExUnit already performed diffing when comparing data types, this new version adds diffing when matching data against a pattern. + +Finally, this release also adds two new guards, `is_struct/1` and `is_map_key/2`, thanks to the strict requirement on Erlang/OTP 21+. + +To learn what else is new, you can read the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.10.0). + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2020-08-20-embedded-elixir-at-farmbot.markdown b/_posts/2020-08-20-embedded-elixir-at-farmbot.markdown new file mode 100644 index 000000000..0471d047f --- /dev/null +++ b/_posts/2020-08-20-embedded-elixir-at-farmbot.markdown @@ -0,0 +1,51 @@ +--- +layout: post +title: Embedded Elixir at Farmbot +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Farmbot. +logo: /images/cases/logos/farmbot.png +tags: embedded nerves +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[FarmBot](https://farm.bot/) is an open-source precision agriculture CNC farming project that includes a robot farming machine, software, and documentation including a farming data repository. FarmBot's machines use IoT technology to make it easy for farmers to remotely manage their gardens. + +Farmbot is built with [Nerves](https://www.nerves-project.org/), an open-source platform and infrastructure to build, deploy, and securely manage your fleet of IoT devices at speed and scale. + +When Connor Rigby, former embedded systems engineer at Farmbot, initially joined the company, his first project was to build a Farmbot application using Ruby. After completing the proof-of-concept, he knew that he needed a way to make the embedded development process more efficient, reliable, and secure. Connor had used Nerves before at a previous company and was a regular open-source contributor to Nerves, so he knew the platform would help him accomplish these goals. + +![Farmbot](/images/cases/bg/farmbot.jpg) + +## How Nerves helped + +Connor brought Nerves to Farmbot by porting the entire proof-of-concept Ruby application he'd created over to Nerves, which he did in his free time over the course of a month, taking him about 20 hours total. He also continued to make open-source contributions to Nerves, helping to structure the networking functionality that is now part of [NervesHub](https://www.nerves-hub.org/), the extensible web service that enables over-the-air firmware update management. + +
+

The biggest benefit of using Nerves is definitely how fast you can get up and running.

+

— Connor Rigby, Embedded Systems Engineer

+
+ +Connor says that the Nerves Platform and what eventually became NervesHub was a great choice for Farmbot because: + +### 1. Nerves supports lean systems and operates well in low-bandwidth areas + +Because Nerves bundles entire applications into relatively small archives in terms of firmware images for full Linux systems, Farmbot can use NervesHub to send over-the-air updates more quickly and users can download them faster. For comparison, an Android update generally clocks in at around 4 GB, but a Nerves update can be packed into as little as 12 MB. + +This is especially helpful for Farmbot users who operate in more remote locations with lower bandwidth and less reliable access to Wi-Fi. When an internet connection is available, NervesHub will connect and check if there's an update, and then prompt the user to install the update. + +### 2. Nerves adds convenience with low overhead + +For devices that are already connected to the internet, connecting to Nerves requires no additional configuration because NervesHub is compatible with the current public key infrastructure for device-to-cloud communication. Since Farmbot already had internet-connected devices when they brought Nerves onboard, they were able to use the same "key" to sign in to NervesHub that they use for their cloud service. + +### 3. Nerves has all the benefits of Elixir and Erlang + +Because it's written in Elixir and built within the Erlang runtime system, Nerves retains the qualities of that language and framework — notably that they are distributed, fault-tolerant, soft real-time, and highly available. Connor also says that with Nerves, it's easy to reason about the things you build with Nerves because you only input what you need into a Nerves application, helping you to avoid unnecessary complexities or unforeseen security vulnerabilities. You can check up on devices as they're running and debug them without disruption to the user experience. + +## The result + +FarmBot now has around 300 devices live in NervesHub, with a different deployment for each of their device models. Nerves is built to scale, so as Farmbot continues to grow its user base and expand their product capabilities, they'll be able to continue developing and releasing reliable firmware updates using Nerves. + +*This case study has first been published on [Nerves' website](https://nerves-project.org/cases/farmbot)*. diff --git a/_posts/2020-09-24-paas-with-elixir-at-Heroku.markdown b/_posts/2020-09-24-paas-with-elixir-at-Heroku.markdown new file mode 100644 index 000000000..bf97dd5f3 --- /dev/null +++ b/_posts/2020-09-24-paas-with-elixir-at-Heroku.markdown @@ -0,0 +1,69 @@ +--- +layout: post +title: PaaS with Elixir at Heroku +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Heroku. +logo: /images/cases/logos/heroku.png +tags: paas phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Heroku](https://www.heroku.com) provides services and tools to build, run, and scale web applications. They enable developers and teams to focus on the design and craft of their apps. Heroku started development back in 2007, focused on the Ruby programming language, and since then, they have expanded to support multiple runtimes, either officially or via buildpacks. + +As the platform grew, their engineering teams also adopted different languages, one of them being Elixir. In this article, we will talk about how two distinct engineering teams at Heroku, the Front-end team and the Vault team, have adopted Elixir. + +![Heroku](/images/cases/bg/heroku.png) + +## First steps with Elixir + +The Vault team was first to use Elixir inside Heroku. Their team is responsible for licensing and financial services, such as invoicing, credit card payments, etc. Most of their services are used internally at Heroku. + +They had to rewrite one of their existing services and that was the perfect occasion to give Elixir a try, since the difficulties and risks with the service were mostly known. The experiment was a success: they deployed and ran their first Elixir application in production. This paved the way to use Elixir more and more. + +Later on, they had a new challenge: they had to audit a large amount of data, and they knew from experience that the Ruby implementation would take too long to finish. Given they were already ramping up their familiarity with Elixir, they chose to apply [Elixir's GenStage](https://github.com/elixir-lang/gen_stage) to the problem, which is a low-level library for data processing, and that took only a couple hours. From this moment on, they were sold on the language and the platform. + +## Tackling operational complexity with Elixir + +The Front-end team shares a similar story: they first used Elixir to solve a well-understood problem and took it forward from there. + +The Front-end engineers are responsible for maintaining all user interfaces: the CLI, the dashboard, and a bunch of backend services that work with data. One of the features they provide to Heroku customers is analytics. + +At first, they were sending their analytics to Mixpanel. However, they had some issues fetching the data, due to cross-domain concerns, and they decided to replace Mixpanel by an in-house Elixir service. The service used [Plug](https://github.com/elixir-plug/plug), a library for building web applications, and had a single endpoint. + +
+

We were having a lot of fun and a lot of luck with it, so we kept doing it.

+

— Micah Woods, Lead Engineer, on migrating to Elixir.

+
+ +They later spent most of a year focused on operational stability, and during this period, they started rewriting part of their Node.js microservices into Elixir. Today they have migrated their numerous Node.js microservices into one main Elixir application with one auxiliary service for authentication. The fact that Elixir was capable of handling everything they threw at it alongside their experience with Erlang's stability - [Heroku's router uses Erlang](https://blog.heroku.com/erlang-in-anger) - allowed them to simplify their operations considerably. + +## Productivity and scalability + +The Front-end team has been using Elixir for two years. The team has 21 engineers: about 4 of them doing Elixir full-time, and 8 engineers altogether doing Elixir here and there. + +The first service that they built with Elixir, the analytics services, receives requests and puts them into an in-memory queue to be processed within the same VM. It handles about 3k to 4k requests per second. 99% of the response times stay within 0-1ms, occasionally 4ms. They use 3 Heroku dynos for fault-tolerance - of course, Heroku uses Heroku for their own infrastructure. + +The main Elixir application uses [the Phoenix web framework](https://phoenixframework.org/) to power the Heroku Dashboard, provide real-time functionality via WebSockets, and support other services. This application runs on 5 Heroku dynos - although their engineering team believes they could probably do with less. Memory consumption is also on the lower side: their biggest dyno uses 256MB. + +The Vault team doing Elixir is only three engineers. Most of their apps are used internally, so they are generally not worried about performance. They continue using Elixir because they feel productive and happy with it. They have also found it is an easier language to maintain compared to their previous experiences. + +## On Phoenix + +Both teams generally use Phoenix for web applications, unless they have a reason not to, which is rare. They acknowledge there is not a performance penalty for using Phoenix and you get a lot out of the box. Phoenix makes it easy to opt-in on the pieces that they need and remove the parts that they don't want. + +They have also found it easier to understand how Phoenix itself works under the hood, especially compared to their previous experiences with other frameworks, such as Ruby on Rails. This knowledge is consistently helping them maintain and update their applications as time passes. + +## Learning Elixir and growing the team + +The growth of both Elixir teams has been mostly organic. Given there are multiple languages in their stack, they often hire for one or another language in particular, and not specifically for Elixir. If the new team members gravitate towards Elixir, they are further encouraged to explore and learn the language. They are also active practitioners of pair programming, so there are many opportunities in their team to learn from each other, rotate pairs, swap projects, and so on. + +According to Matthew Peck, "the paradigm shift from Object-Oriented languages to Functional Programming was our biggest challenge when first learning Elixir". However, the team agrees the investment was worth it: "Learning Elixir has made us better programmers. We have found that immutability made our code more readable, easier to test, and simpler to make concurrent. Now when we go back to an Object-Oriented language, we are thinking about how we can apply the same concepts there" - said Mike Hagerdon. + +Amanda Dolan added some remarks on Elixir's capabilities for writing concurrent and fault-tolerant applications: "One other challenge when learning Elixir is fully grasping concurrency and the Erlang/OTP patterns". Some of them felt it took longer to master those concepts than they first expected. + +Taylor Mock has his take on the challenges teams may face when adopting Elixir: "Another difference between Elixir and our previous stacks, Ruby and Node.js, is in the ecosystems". They were initially concerned that the Elixir ecosystem would lack when it comes to third-party tools, but that was not what they saw. Taylor continues: "We found out that we can get really far with the concepts and mechanisms that the language itself provides. This shift can be scary, but we are now past it, and we find ourselves with leaner applications and fewer dependencies". + +Overall, both teams found the language itself quite approachable. Given they started with a small proof of concept, they were able to tackle their concerns in regards to adoption, development, and deployment as they moved forward. Historically Heroku also has had much success with Erlang, and that has contributed to the success adopting Elixir has seen inside Heroku. diff --git a/_posts/2020-10-06-elixir-v1-11-0-released.markdown b/_posts/2020-10-06-elixir-v1-11-0-released.markdown new file mode 100644 index 000000000..7aaaefefb --- /dev/null +++ b/_posts/2020-10-06-elixir-v1-11-0-released.markdown @@ -0,0 +1,225 @@ +--- +layout: post +title: Elixir v1.11 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.11 is out with improvements to the compiler and tighter integration with Erlang/OTP +--- + +Over the last releases, the Elixir team has been focusing on the compiler, both in terms of catching more mistakes at compilation time and making it faster. Elixir v1.11 has made excellent progress on both fronts. This release also includes many other goodies, such as tighter Erlang integration, support for more guard expressions, built-in datetime formatting, and other calendar enhancements. + +During this period, we have also started [publishing a series of production cases](/cases.html) on our website, featuring Elixir's usage at [Farmbot](/blog/2020/08/20/embedded-elixir-at-farmbot/) and [Heroku](/blog/2020/09/24/paas-with-elixir-at-Heroku/), with many more cases coming soon. + +For now, let's focus on what's new in Elixir v1.11. + +## Tighter Erlang integration + +On the footsteps of v1.10, we have further integrated with Erlang's new logger by adding four new log levels: `notice`, `critical`, `alert`, and `emergency`, matching all log levels found in the Syslog standard. The `Logger` module now supports structured logging by passing maps and keyword lists to its various functions. It is also possible to specify the log level per module, via the [`Logger.put_module_level/2`](https://hexdocs.pm/logger/Logger.html#put_module_level/2) function. Log levels per application will be added in future releases. + +IEx also has been improved to show the documentation for Erlang modules directly from your Elixir terminal. For example, here is a clip of me accessing the documentation for [Erlang's gen_server module](erlang.org/doc/man/gen_server.html): + + + + This works with Erlang/OTP 23+ and requires Erlang modules to have been compiled with documentation chunks. A huge thank you to the Erlang/OTP team and the Documentation Working Group of the [Erlang Ecosystem Foundation](https://erlef.org/) for making this possible. + +## Compiler checks: application boundaries + +Elixir v1.11 builds on top of the recently added compilation tracers to track application boundaries. From this release, Elixir will warn if you invoke a function from an existing module but this module does not belong to any of your listed dependencies. + +These two conditions may seem contradictory. After all, if a module is available, it must have come from a dependency. This is not true in two scenarios: + + * Modules from Elixir and Erlang/OTP are always available - even if their applications are not listed as a dependency + + * In an umbrella project, because all child applications are compiled within the same VM, you may have a module from a sibling project available, even if you don't depend on said sibling + +This new compiler check makes sure that all modules that you invoke are listed as part of your dependencies, emitting a warning like below otherwise: + +```text +:ssl.connect/2 defined in application :ssl is used by the current +application but the current application does not directly depend +on :ssl. To fix this, you must do one of: + + 1. If :ssl is part of Erlang/Elixir, you must include it under + :extra_applications inside "def application" in your mix.exs + + 2. If :ssl is a dependency, make sure it is listed under "def deps" + in your mix.exs + + 3. In case you don't want to add a requirement to :ssl, you may + optionally skip this warning by adding [xref: [exclude: :ssl] + to your "def project" in mix.exs +``` + +This comes with extra benefits in umbrella projects, as it requires applications to depend on the siblings they depend on, which will fail if there are any cyclic dependencies. + +## Compiler checks: data constructors + +In Elixir v1.11, the compiler also tracks structs and maps fields across a function body. For example, imagine you wanted to write this code: + +```elixir +def drive?(%User{age: age}), do: age >= 18 +``` + +If there is either a typo on the `:age` field or the `:age` field was not yet defined, the compiler will fail accordingly. However, if you wrote this code: + +```elixir +def drive?(%User{} = user), do: user.age >= 18 +``` + +The compiler would not catch the missing field and an error would only be raised at runtime. With v1.11, Elixir will track the usage of all maps and struct fields within the same function, emitting warnings for cases like above: + +```text +warning: undefined field `age` in expression: + + # example.exs:7 + user.age + +expected one of the following fields: name, address + +where "user" was given the type %User{} in: + + # example.exs:7 + %User{} = user + +Conflict found at + example.exs:7: Check.drive?/1 +``` + +The compiler also checks binary constructors. Consider you have to send a string over the wire with length-based encoding, where the string is prefixed by its length, up to 4MBs. Your initial attempt may be this: + +```elixir +def run_length(string) when is_binary(string) do + <> +end +``` + +However, the code above has a bug. Each segment given between `<<>>` must be an integer, unless specified otherwise. With Elixir v1.11, the compiler will let you know so: + +```text +warning: incompatible types: + + binary() !~ integer() + +in expression: + + <> + +where "string" was given the type integer() in: + + # foo.exs:4 + <> + +where "string" was given the type binary() in: + + # foo.exs:3 + is_binary(string) + +HINT: all expressions given to binaries are assumed to be of type integer() +unless said otherwise. For example, <> assumes "expr" is an integer. +Pass a modifier, such as <> or <>, to change the +default behaviour. + +Conflict found at + foo.exs:4: Check.run_length/1 +``` + +Which can be fixed by adding `::binary` to the second component: + +```elixir +def run_length(string) when is_binary(string) do + <> +end +``` + +While some of those warnings could be automatically fixed by the compiler, future versions will also perform those checks across functions and potentially across modules, where automatic fixes wouldn't be desired (nor possible). + +## Compilation time improvements + +Elixir v1.11 features many improvements to how the compiler tracks file dependencies, such that touching one file causes less files to be recompiled. In previous versions, Elixir tracked three types of dependencies: + + * compile time dependencies - if A depends on B at compile time, such as by using a macro, whenever B changes, A is recompiled + * struct dependencies - if A depends on B's struct, whenever B's struct definition changed, A is recompiled + * runtime dependencies - if A depends on B at runtime, A is never recompiled + +However, because dependencies are transitive, if A depends on B at compile time and B depends on C at runtime, A would depend on C at compile time. Therefore, it is very important to reduce the amount of compile time dependencies. + +Elixir v1.11 replaces "struct dependencies" by "exports dependencies". In other words, if A depends on B, whenever B public's interface changes, A is recompiled. B's public interface is made by its struct definition and all of its public functions and macros. + +This change allows us to mark `import`s and `require`s as "exports dependencies" instead of "compile time" dependencies. This simplifies the dependency graph considerably. For example, [in the Hex.pm project](https://github.com/hexpm/hexpm), changing the `user.ex` file in Elixir v1.10 would emit this: + +```text +$ touch lib/hexpm/accounts/user.ex && mix compile +Compiling 90 files (.ex) +``` + +In Elixir v1.11, we now get: + +```text +$ touch lib/hexpm/accounts/user.ex && mix compile +Compiling 16 files (.ex) +``` + +To make things even better, Elixir v1.11 also introduces a more granular tracking for umbrella projects (and path dependencies in general). In previous versions, a module from a sibling application would always be treated as a compile time dependency. This often meant that changing an application would cause many modules in sibling applications to recompile. Elixir v1.11 will tag modules from dependencies as exports whenever possible, yielding dramatic improvements in those cases. + +To round up the list of compiler enhancements, the `--profile=time` option added in Elixir v1.10 now also includes the time to compile each individual file. For example, in the Plug project, one can now get: + +```text +[profile] lib/plug/conn.ex compiled in 935ms +[profile] lib/plug/ssl.ex compiled in 147ms (plus 744ms waiting) +[profile] lib/plug/static.ex compiled in 238ms (plus 654ms waiting) +[profile] lib/plug/csrf_protection.ex compiled in 237ms (plus 790ms waiting) +[profile] lib/plug/debugger.ex compiled in 719ms (plus 947ms waiting) +[profile] Finished compilation cycle of 60 modules in 1802ms +[profile] Finished group pass check of 60 modules in 75ms +``` + +While implementing those features, we have also made the `--long-compilation-threshold` flag more precise. In previous versions, `--long-compilation-threshold` would consider both the time a file spent to compile and the time spent waiting on other files. In Elixir v1.11, it considers only the compilation time. This means less false positives and you can now effectively get all files that take longer than 2s to compile, in execution time, by passing `--long-compilation-threshold 2`. + +## `config/runtime.exs` and `mix app.config` + +Elixir v1.9 introduced a new configuration file called `config/releases.exs`. However, this new configuration file was executed only during releases. For those not familiar with releases, a release is a self-contained artifact with the Erlang VM, Elixir and your application, ready to run in production. + +This new configuration file was considered a very useful addition to releases. Therefore, we are also introducing `config/runtime.exs`, which is executed after the code compilation on all environments (dev, test, and prod) - for both Mix and releases. Our goal is to provide a better runtime configuration experience to developers, in contrast to our current configuration system which has been mostly compile-time centric. + +`config/runtime.exs` works the same as any other configuration file in Elixir. However, given `config/runtime.exs` is meant to run in production systems, where our `Mix` build tool is not available, developers must not use [`Mix.env()`](https://hexdocs.pm/mix/Mix.html#env/0) or [`Mix.target()`](https://hexdocs.pm/mix/Mix.html#target/0) in `config/runtime.exs`. Instead, they must use the new `config_env()` and `config_target()`, which have been added to the [`Config`](https://hexdocs.pm/elixir/Config.html) module. + +While `config/releases.exs` will continue to be supported, developers can migrate to `config/runtime.exs` without loss of functionality. For example, a `config/releases.exs` file such as this one + +```elixir +# config/releases.exs +import Config + +config :foo, ... +config :bar, ... +``` + +could run as is as `config/runtime.exs`. However, given `config/runtime.exs` runs in all environments, you may want to restrict part of your configuration to the `:prod` environment: + +```elixir +# config/runtime.exs +import Config + +if config_env() == :prod do + config :foo, ... + config :bar, ... +end +``` + +If both files are available, releases will pick the now preferred `config/runtime.exs` instead of `config/releases.exs`. + +To wrap it all up, `Mix` also includes a new task called [`mix app.config`](https://hexdocs.pm/mix/Mix.Tasks.App.Config.html). This task loads all applications and configures them, without starting them. Whenever you write your own Mix tasks, you will typically want to invoke either `mix app.start` or `mix app.config` before running your own code. Which one is better depends if you want your applications running or only configured. + +## Other improvements + +Elixir v1.11 adds the `is_struct/2`, `is_exception/1`, and `is_exception/2` guards. It also adds support for the `map.field` syntax in guards. + +The Calendar module ships with a new [`Calendar.strftime/3`](https://hexdocs.pm/elixir/Calendar.html#strftime/3) function, which provides datetime formatting based on the `strftime` format. The [`Date`](https://hexdocs.pm/elixir/Date.html) module got new functions for working with weeks and months, such as `Date.beginning_of_month/1` and `Date.end_of_week/2`. Finally, all calendar types got conversion functions from and to gregorian timestamps, such as `Date.from_gregorian_days/2` and `NaiveDateTime.to_gregorian_seconds/1`. + +Finally, to bring visibility to the compiler tracking improvements described in previous sections, we have also added new features to [`mix xref`](https://hexdocs.pm/mix/Mix.Tasks.Xref.html). `mix xref` is a task that describes cross-references between files in your projects and can be helpful to diagnose large compilation cycles in projects. + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.11.0). + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2020-10-08-real-time-communication-at-scale-with-elixir-at-discord.markdown b/_posts/2020-10-08-real-time-communication-at-scale-with-elixir-at-discord.markdown new file mode 100644 index 000000000..ac4f42ef8 --- /dev/null +++ b/_posts/2020-10-08-real-time-communication-at-scale-with-elixir-at-discord.markdown @@ -0,0 +1,70 @@ +--- +layout: post +title: Real time communication at scale with Elixir at Discord +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Discord. +flagship: true +logo: /images/cases/logos/discord.png +tags: real-time genstage otp +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +Founded in 2015 by Jason Citron and Stan Vishnevskiy, [Discord](https://discord.com/) is a permanent, invite-only space for your communities and friends, where people can hop between voice, video, and text, depending on how they want to talk, letting them have conversations in a very natural or authentic way. Today, the service has over 100 million monthly active users from across the globe. Every day people spend 4 billion minutes in conversation on Discord servers, across 6.7 million active servers / communities. + +From day one, Discord has used Elixir as the backbone of its chat infrastructure. When Discord first adopted the language, they were still working on building a viable business, with many questions and challenges in front of them. Elixir played a crucial role in giving them the desired technological flexibility to grow the company and also became the building block that would allow their systems to run on a massive scale. + +![Discord](/images/cases/bg/discord.jpg) + +## Starting technologies + +Back in 2015, Discord chose two main languages to build their infrastructure: Elixir and Python. Elixir was initially picked to power the WebSocket gateway, responsible for relaying messages and real-time replication, while Python powered their API. + +Nowadays, the Python API is a monolith while the Elixir stack contains 20 or so different services. These architectural choices do not represent a dichotomy between the languages but rather a pragmatic decision. Mark Smith, from the Discord team, explains it succinctly: "given the Elixir services would handle much bigger traffic, we designed them in a way where we could scale each service individually." + +Discord has also explored other technologies along the way, Go and Rust being two examples, with distinct outcomes. While Discord completely phased out Go after a short foray, Rust has proven to be an excellent addition to their toolbox, boosted by its ability to play well with Elixir and Python. + +## Communication at scale + +Effective communication plays an essential role when handling millions of connected users concurrently. To put things into perspective, some of Discord's most popular servers, such as those dedicated to Fortnite and Minecraft, are nearing six hundred thousand users. At a given moment, it is not unlikely to encounter more than two hundred thousand active users in those servers. If someone changes their username, Discord has to broadcast this change to all connected users. + +Overall, Discord's communication runs at impressive numbers. They have crossed more than 12 million concurrent users across all servers, with more than 26 million WebSocket events to clients per second, and Elixir is powering all of this. + +
+

In terms of real time communication, the Erlang VM is the best tool for the job.

+

— Jake Heinz, Lead Software Engineer

+
+ +When we asked their team "Why Elixir?", Jake Heinz gave a straight-forward answer: "In terms of real time communication, the Erlang VM is the best tool for the job. It is a very versatile runtime with excellent tooling and reasoning for building distributed systems". Technologically speaking, the language was a natural fit. However, Elixir was still a bet back in 2015: "Elixir v1.0 had just come out, so we were unsure in which direction the language would go. Luckily for us, we have been pleased with how the language has evolved and how the community shaped up." + +## The chat infrastructure team + +To power their chat messaging systems, Discord runs a cluster with 400-500 Elixir machines. Perhaps, the most impressive feat is that Discord's chat infrastructure team comprises five engineers. That's right: five engineers are responsible for 20+ Elixir services capable of handling millions of concurrent users and pushing dozens of millions of messages per second. + +Discord also uses Elixir as the control plane of their audio and video services, also known as signaling, which establishes communication between users. C++ is then responsible for media streaming, a combination that altogether runs on 1000+ nodes. + +The Elixir services communicate between them using Distributed Erlang, the communication protocol that ships as part of the Erlang Virtual Machine. By default, Distributed Erlang builds a fully meshed network, but you can also ask the Erlang VM to leave the job of outlining the topology up to you, by setting the aptly named `-connect_all false` flag. The Discord team sets this option to assemble a partially meshed network with [etcd](https://etcd.io/) being responsible for service discovery and hosting shared configuration. + +The chat infrastructure developers are not the only ones touching the Elixir codebases. According to Mark Smith, this is an important part of Discord's culture: "We don't work in silos. So a Python developer may have to work on the Elixir services when building a new feature. We will spec out the feature together, figure out the scalability requirements, and then they will work on a pull request, which we will review and help them iterate on it." + +## Community and challenges + +To run at this scale, Discord learned how to leverage the Erlang VM's power, its community, and when to recognize challenges that require them to reach for their own solutions. + +For example, Discord uses [Cowboy](https://github.com/ninenines/cowboy/) for handling WebSocket connections and TCP servers. To manage data bursts and provide load regulation, such as back-pressure and load-shedding, they use [GenStage](https://github.com/elixir-lang/gen_stage), which they have [discussed in detail in the past](https://discord.com/blog/how-discord-handles-push-request-bursts-of-over-a-million-per-minute-with-elixirs-genstage). + +Other times, the efforts of the company and the community go hand in hand. That was the case when Discord used [the Rustler project](https://github.com/rusterlium/rustler), which provides a safe bridge between Elixir and Rust, to [scale to 11 million concurrent users](https://discord.com/blog/using-rust-to-scale-elixir-for-11-million-concurrent-users). They used the Rustler to hook a custom data structure built in Rust directly into their Elixir services. + +However, the team has made abundantly clear that the powerhouse is the Erlang platform. Every time they had to push their stack forward, they never felt cornered by the technology. Quite the opposite, their engineers could always build efficient solutions that run at Discord's scale, often in a few hundred lines of code. Discord frequently gives these projects back to the community, as seen in [Manifold](https://github.com/discord/manifold) and [ZenMonitor](https://github.com/discord/zen_monitor). + +The Discord team also adapted quickly when things went wrong. For instance, they attempted twice to use [Mnesia](https://www.erlang.org/doc/man/mnesia.html) in production —a database that ships as part of Erlang's standard library. They tried Mnesia in persistent and in-memory modes, and the database nodes would often fall behind in failure scenarios, sometimes being unable to ever catch up. Eventually they ditched Mnesia altogether and built the desired functionality with Erlang's builtin constructs, such as GenServer and ETS. Nowadays, they resolve these same failure scenarios within 2-3 seconds. + +## Mastering Elixir + +None of the chat infrastructure engineers had experience with Elixir before joining the company. They all learned it on the job. Team members Matt Nowack and Daisy Zhou report initially struggling to understand how all of their services communicate. Matt adds: "In the beginning, it was hard to accept all of the guarantees that Erlang VM provides. I'd worry about data races and concurrency issues that were impossible to happen". Eventually, they took these guarantees to heart and found themselves more productive and more capable of relying on the platform and its tools. Matt continues: "The introspection tools the Erlang VM provides is the best in class. We can look at any VM process in the cluster and see its message queue. We can use the remote shell to connect to any node and debug a live system. All of this has helped us countless times." + +Running at Discord's scale adds its own dimension to mastering the language, as they need to familiarize with the abstractions for providing concurrency, distribution, and fault-tolerance. Nowadays, frameworks such as Nerves and Phoenix handle these concerns for developers, but the underlying building blocks are always available for engineers assembling their own stack, such as the Discord team. + +In the end, Jake summarized how crucial Elixir and the Erlang VM have been at Discord and how it affected him personally: "What we do in Discord would not be possible without Elixir. It wouldn't be possible in Node or Python. We would not be able to build this with five engineers if it was a C++ codebase. Learning Elixir fundamentally changed the way I think and reason about software. It gave me new insights and new ways of tackling problems." diff --git a/_posts/2020-10-27-delivering-social-change-with-elixir-at-change.org.markdown b/_posts/2020-10-27-delivering-social-change-with-elixir-at-change.org.markdown new file mode 100644 index 000000000..8e58ec64c --- /dev/null +++ b/_posts/2020-10-27-delivering-social-change-with-elixir-at-change.org.markdown @@ -0,0 +1,85 @@ +--- +layout: post +title: Delivering social change with Elixir at Change.org +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Change.org. +flagship: true +logo: /images/cases/logos/change.png +tags: social broadway +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Change.org](https://change.org/) is a social change platform, with over 400 million users worldwide. Two years ago, their engineering team faced a challenge to migrate their messaging system from an external vendor to an in-house solution, to reduce costs and gain flexibility. + +This article will discuss how they approached this problem, why they chose Elixir, and how their system grew to deliver more than 1 billion emails per month. Change.org is also [hiring Elixir engineers to join their team](https://www.change.org/careers). + +![Change.org](/images/cases/bg/change.png) + +## The path to Elixir + +The first step for Change.org's engineering team was to outline the requirements for their system. The system would receive millions of events, such as campaign updates, new petitions, and more, and it should send emails to all interested parties whenever appropriate. They were looking for an event-driven solution at its core, in which concurrency and fault-tolerance were strong requirements. + +The next stage was to build proofs-of-concept in different programming languages. Not many companies can afford this step, but Change.org's team knew the new system was vital to their business and wanted to be thorough in their analysis. + +Around this time, John Mertens, Director of Engineering, was coming back from parental leave. He used this opportunity to catch up with different technologies whenever possible. That's when he stumbled upon [José Valim's presentation at Lambda Days](https://www.youtube.com/watch?v=XPlXNUXmcgE), which discussed two libraries in the Elixir ecosystem: [GenStage](https://github.com/elixir-lang/gen_stage) and [Flow](https://github.com/dashbitco/flow). + +They developed prototypes in four technologies: JRuby, Akka Streams, Node.js, and Elixir. The goal was to evaluate performance, developer experience, and community support for their specific use cases. Each technology had to process 100k messages as fast as possible. John was responsible for the Elixir implementation and put his newly acquired knowledge to use. + +After two evaluation rounds, the team chose to go ahead with Elixir. Their team of 3 engineers had 18 months to replace the stack they had been using for the last several years with their own Elixir implementation. + +## Learning Elixir + +When they started the project, none of the original team members had prior experience with Elixir. Only Justin Almeida, who joined when the project had been running by six months, had used Elixir before. + +Luckily, the team felt supported by the different resources available in the community. John recalls: "We were in one of our early meetings discussing how to introduce Elixir into our stack when Pragmatic Programmers announced the [Adopting Elixir](https://pragprog.com/titles/tvmelixir/adopting-elixir/) book, which was extremely helpful in answering many of our questions." + +## The new system + +The team developed three Elixir applications to replace the external vendor. The first application processes all incoming events to decide whether an email should go out and to whom. + +The next application is the one effectively responsible for dispatching the emails. For each message, it finds the appropriate template as well as the user locale and preferences. It then assembles the email and delivers it with the help of a Mail Transfer Agent (MTA). + +The last application is responsible for analytics. It receives webhook calls from the MTA with batches of different events, which are processed and funneled into their data warehouse for later use. + +After about four months, they put the new system in production. While Change.org has dozens of different email templates, the initial deployment handled a single and straight-forward case: password recovery. + +Once the new system was in production, they continued to migrate different use cases to the system, increasing the numbers of handled events and delivered emails day after day. After one year, they had completed the migration ahead of schedule. + +## Handling spikes and load regulation + +Today, those applications run on a relatively small number of nodes. The first two applications use 6 to 8 nodes, while the last one uses only two nodes. + +John explains they are over-provisioned because spikes are relatively frequent in the system: "for large campaigns, a single event may fan out to thousands or hundreds of thousands of emails." + +The team was kind enough to share some of their internal graphs. In the example below, you can see a spike of over 10 million messages coming to the system: + +![Usage at Change.org](/images/cases/bg/change-graph.png) + +Once this burst happens, all nodes max their CPUs, emitting around 3000 emails per second until they drain the message queue. The whole time memory usage remains at 5%. + +The back-pressure provided by the [GenStage](https://github.com/elixir-lang/gen_stage) library played a crucial role in the system’s performance. Since those applications fetch events from message queues, process them, and submit them into third-party services, they must avoid overloading any part of the stack. GenStage addresses this by allowing the different components, called stages in the library terminology, to communicate how much data they can handle right now. For example, if sending messages to the MTA is slower than usual, the system will naturally get fewer events from the queue. + +Another essential feature of the system is to work in batches. Receiving and sending data is more efficient and cost-effective if you can do it in groups instead of one-by-one. John has given [a presentation at ElixirConf Europe sharing the lessons learned from their first trillion messages](https://www.youtube.com/watch?v=t46L9RKmlNo). + +The activity on Change.org has grown considerably over the last year too. The systems have coped just fine. Justin remarks: "everything has been working so well that some of those services are not really on our minds." + +## Working with the ecosystem + +Change.org has relied on and contributed to the ecosystem whenever possible. During the migration, both old and new systems had to access many shared resources, such as [HAML templates](https://haml.info/), Ruby's I18N configuration files, and [Sidekiq's background queues](https://sidekiq.org/). Fortunately, they were able to find compatible libraries in the Elixir ecosystem, respectively [calliope](https://github.com/nurugger07/calliope), [linguist](https://github.com/change/linguist), and [exq](https://github.com/akira/exq). + +Nowadays, some of those libraries have fallen out of flavor. For example, the community has chosen gettext for internationalization, as it is a more widely accepted format. For this reason, Change.org has stepped in and taken ownership of the linguist library. + +As Change.org adopted Elixir, the ecosystem grew to better support their use cases too. One recent example [is the Broadway library](https://github.com/dashbitco/broadway), which makes it easy to assemble data pipelines. John explains: "Broadway builds on top of GenStage, so it provides the load regulation, concurrency, and fault-tolerance that we need. It also provides batching and partitioning, which we originally had to build ourselves. For new projects, Broadway is our first choice for data ingestion and data processing." + +## Elixir as the default stack + +As projects migrate to Elixir, Elixir has informally become the default stack at Change.org for backend services. Today they have more than twenty projects. The engineering team has also converged on a common pattern for services in their event driven architecture, built with Broadway and Phoenix. + +In a nutshell, they use Broadway to ingest, aggregate, and store events in the database. Then they use Phoenix to expose this data, either through APIs, as analytics or as tooling for their internal teams. + +One recent example is [Change.org's Bandit service](https://medium.com/making-change-org/our-elixir-bandit-service-e2b6af6eebc4). The service provides a Phoenix API that decides which copy to present to users in various parts of their product. As users interact with these copies, data is fed into the system and analyzed in batches with Broadway. They use this feedback to optimize and make better choices in the future. + +The team has also grown to ten Elixir developers thanks to the multiple training and communities of practice they have organized internally. Change.org is also looking for Elixir backend engineers, as they aim to bring experience and diversity to their group. Interested developers can [learn more about these opportunities on their website](https://www.change.org/careers). diff --git a/_posts/2020-11-17-real-time-collaboration-with-elixir-at-slab.markdown b/_posts/2020-11-17-real-time-collaboration-with-elixir-at-slab.markdown new file mode 100644 index 000000000..e699b459a --- /dev/null +++ b/_posts/2020-11-17-real-time-collaboration-with-elixir-at-slab.markdown @@ -0,0 +1,70 @@ +--- +layout: post +title: Real-time collaboration with Elixir at Slab +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Slab. +logo: /images/cases/logos/slab.png +tags: collab phoenix otp +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Slab](https://slab.com/) is a knowledge base and team wiki that democratizes knowledge. Jason Chen started Slab in August 2016, after picking Elixir and Phoenix as the best tools to build real-time collaborative applications. The company has grown to 6 engineers since then, distributed worldwide, and relied upon by more than 7000 companies and customers like Asana, Discord, and Glossier. If you are interested in helping companies become a source of learning and purpose, especially during these times where remote collaboration is essential, [Slab is hiring](https://slab.com/jobs?ref=elixir). + +![Slab](/images/cases/bg/slab.png) + +## Why Elixir? + +Slab was not the first time Jason wrote a collaborative web app. He had previous practice building them in Rails and Node.js and he believed there was a lot to improve in the development experience, especially when it came to working with WebSockets. Both technologies were also troublesome in production, as the team faced issues scaling them vertically and horizontally. + +
+

I wanted a framework with the same developer experience as Django and Rails, but one that was designed for real-time applications.

+

— Jason Chen, CEO, on the Phoenix web framework

+
+ +Jason doesn't consider himself a person who is always looking for new things, but he knew he would have to survey the options around him when starting Slab. During this period, he explored two main languages: Go and Elixir. In the end, Jason chose Elixir, thanks to [the Phoenix web framework](https://phoenixframework.org/): "I was looking for a framework that offered a complete toolset for building web apps. I was not interested in making low-level choices, such as which ORM to use, which library to pick for parsing requests, etc. I wanted a framework with the same developer experience as Django and Rails, but one that was designed for real-time applications". + +Jason gave himself two weeks to build a proof of concept. He wrote a collaborative blog, where multiple users could write a post simultaneously, and comments were added in real-time — all while learning Elixir and the Phoenix framework. + +The trial went better than expected, and Jason's journey with Slab had officially begun. + +## Growing with the platform + +Shortly after, Slab was in a private beta with a handful of companies as users. For each major feature they had along the way, Elixir and Phoenix provided the building blocks for it. When they implemented real-time comments, they used Phoenix Channels and Phoenix PubSub. The pattern goes on: "for asynchronous processing, we simply use [Elixir tasks](https://hexdocs.pm/elixir/Task.html)". Later on, to track users editing a document and give each a different cursor color, they used [Phoenix Presence](https://hexdocs.pm/phoenix/Phoenix.Presence.html), a tool that no other web framework offers out-of-the-box. + +Another leap in Jason's journey with Slab and Elixir was when he had to learn Erlang/OTP, a group of behaviors that ship as part of Erlang's standard library for building distributed and fault-tolerant applications. + +To improve the real-time collaborative editor that is part of Slab, Jason implemented [Operational Transformation](https://en.wikipedia.org/wiki/Operational_transformation). The client runs in the browser, implemented with [React](https://reactjs.org/). As users make changes to the text, their diffs are sent to the server, which arbitrates these updates and synchronizes them across the various clients. + +Tackling the synchronization problem is not trivial, especially when the application is running on multiple nodes. Here is the challenge they faced. Imagine user Alice has a WebSocket connection to node X and user Bob is connected to node Y. Both Alice and Bob are working on the same text. How can Slab guarantee that changes from both users are applied, so both see the same document once done editing? + +One could try to solve this problem by keeping the server stateless. Every time the server receives a diff from the client, the server would read the document from the database, apply the changes, normalize the result, and broadcast the clients' updates. With this approach, the issue is that loading the text from the database on every client update would quickly become expensive, especially as they grow in size. Response times would become higher and the user experience would degrade. + +When working with Node.js, Jason tried a different approach. If Alice and Bob were writing to the same document, a load balancer would guarantee that both would be routed to the same node. After trying out both Apache and Nginx, he implemented the balancer in Node.js. The overall solution was time-consuming to get right and introduced operational complexities. + +Luckily, these problems are the bread and butter of Erlang/OTP. Jason knew he needed a stateful abstraction to keep this state on the server. He had already heard about the options the platform provides, but he was unsure which one to pick. Jason recalls: "I remember asking the community if I should use an [Agent](https://hexdocs.pm/elixir/Agent.html) or a [GenServer](https://hexdocs.pm/elixir/GenServer.html) and everyone was really helpful in providing guidance." They quickly landed on GenServer as their tool of choice. + +By default, both GenServer and Agents are local to each node. However, they also support the `:global` option, which registers a given name across the cluster. To use this option, they need the Erlang distribution, which they were already using for Phoenix PubSub and Presence, so this was a straight-forward change. This guarantees both Alice and Bob talk to the same GenServer, regardless if they joined node X or node Y. + +Later on, when running the system in production, the platform continued to impress him. Every time they increased the machine resources, they could see the runtime efficiently using everything it had available, without changes to the code. + +## Learning and tools + +There are other few notable tools in Slab's stack. + +Back in 2017, they migrated to GraphQL [powered by Elixir's Absinthe](http://absinthe-graphql.org/). There were concerns about adopting the query language, as it was a relatively new technology. Still, they felt it would address a real issue: they had different components in the application needing distinct data, and managing all of these possible combinations was becoming complex. This was one of the main problems GraphQL was designed to solve. + +They are also running on Google Cloud with Kubernetes (K8s), and, as many Elixir engineers, they wondered [how the Erlang VM fit in a world with Docker and K8s](https://dashbit.co/blog/kubernetes-and-the-erlang-vm-orchestration-on-the-large-and-the-small). Today they run on 6 nodes, 5 of them running application code. The sixth one handles [cron jobs](https://en.wikipedia.org/wiki/Cron) and stays on standby for new deployments. They use [the peerage library](https://github.com/mrluc/peerage) to establish Distributed Erlang connections between the nodes. + +
+

We really value Elixir's ability to build complex systems using fewer moving parts. The code is simpler, and the system is easier to operate.

+

— Sheharyar Naseer, engineer

+
+ +Overall the Slab team aims to keep the number of dependencies low, something they believe is made possible by the platform and positively impacts onboarding new developers. Sheharyar Naseer, a member of their engineering team, explains: "We really value Elixir's ability to build complex systems using fewer moving parts. The code is simpler, and the system is easier to operate, making both experienced and new engineers more productive. We ran in production for more than 3 years without resorting to Redis. We just recently added it because we wanted our caches to survive across deployments. Many other stacks impose technologies like Redis from day one." + +This approach also yields benefits when updating libraries. Sheharyar continues: "For the most part, upgrading Erlang, Elixir, and Phoenix is straight-forward. We go through the CHANGELOG, which always emphasizes the main changes we need to perform, and we have a pull request ready after one or two hours. The only time we could not upgrade immediately was when Erlang/OTP removed old SSL ciphers, which broke our HTTP client and we caught it early on during development." + +When onboarding engineers, Slab recommends them different books and video courses — many of which you can find [in our learning resources page](/learning.html) — so they have the flexibility to choose a medium they are most productive with. New engineers also work on Slab itself and receive guidance through pull requests. They start with small tasks, usually in the client and GraphQL layers, and slowly tackle more complex problems around the database and Erlang/OTP. If you are interested in improving remote collaboration, [learn more about their opportunities on their website](https://slab.com/jobs?ref=elixir). diff --git a/_posts/2020-12-10-integrating-travel-with-elixir-at-duffel.markdown b/_posts/2020-12-10-integrating-travel-with-elixir-at-duffel.markdown new file mode 100644 index 000000000..44f8c66cb --- /dev/null +++ b/_posts/2020-12-10-integrating-travel-with-elixir-at-duffel.markdown @@ -0,0 +1,44 @@ +--- +layout: post +title: Integrating travel with Elixir at Duffel +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Duffel. +logo: /images/cases/logos/duffel.png +tags: api integration xml +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Duffel](https://duffel.com/) is building the new software backbone of the travel industry. Their first product is the Flights API, a developer-friendly platform that enables any business to instantly search flights, make bookings, reserve seats, and sell paid extras like checked bags. Duffel is connected to more than 20 of the world’s largest airlines, including American Airlines, Emirates, and Lufthansa. The company was founded in November 2017 and in 2019 it opened a private beta of their API and has raised $56M total in funding. It now has 40 employees across its offices in London and New York. This article discusses how Duffel has used Elixir as their technology of choice to modernize an industry built on old standards and outdated stacks. If you are interested in shaping the future of travel, [Duffel is hiring](https://duffel.com/careers). + +![Duffel](/images/cases/bg/duffel.png) + +## Why Elixir? + +Today, to access flights and prices from airlines, companies have to go through a Global Distribution System (GDS), often using decades-old systems such as Amadeus and Sabre. Steve Domin, Duffel's founder / CEO, explains: "The airline industry runs on a legacy data exchange standard called [EDIFACT standard](https://en.wikipedia.org/wiki/EDIFACT) and only recently moved to a ‘modern’ SOAP/XML world. Any integration work with a GDS or an airline is always scheduled to take months, and this creates a very high barrier to entry for new businesses." + +At its heart, Duffel is building the new operating system for travel. A single request to Duffel's API may translate into a chain of dozens of requests to different airlines. The response of each request needs to be parsed, normalized, and potentially be hydrated with more outgoing requests. All of this while managing slow responses, timeouts, large data payloads, and more. These challenges made it clear to Steve that Elixir would be a great fit: "We are building a highly concurrent platform with intensive data trafficking. From day one, it was clear the Erlang VM would be a great fit, as it was designed for telecommunication with similar requirements in mind." They chose the Erlang VM, alongside [the Phoenix web framework](https://phoenixframework.org/) and [the Ecto database library](https://github.com/elixir-ecto/ecto) as their stack to launch their initial JSON API. They leverage Elixir's standard library for most of their concurrent work and [the Saxy library for XML parsing](https://github.com/qcam/saxy). + +## Growing with Open Source + +When Steve co-founded the company in November 2017, he already had plenty of experience with Elixir. Steve started using the language before it reached 1.0, back in 2013. He started his journey by hacking on [Dynamo](https://github.com/devinus/dynamo), Phoenix's ancestor, and eventually introduced Elixir at his previous company, by using it for an internal project. He also organized meet-ups in London and contributed to Open Source projects, including some of his own, such as [Swoosh](https://github.com/swoosh/swoosh). + +The founders joined [Y Combinator](https://www.ycombinator.com/) in Summer 2018. Once they came back to London, they hired Alan Kennedy as their first engineer. Alan first heard about Elixir when he and Steve were colleagues at GoCardless. Alan kept an eye on it but never actively used it until he joined Duffel. Alan recalls struggling to jump from a language that promotes mutability to an immutable language like Elixir. Once everything clicked, he acknowledged the new programming model is conceptually much simpler. + +Since then, the company has grown with a mixture of fresh and experienced engineers, including nearly 70% of the engineering organisation programming in Elixir. + +Johanna Larsson is one of the most recent engineers to join Duffel. She had already spoken at Elixir Conferences and made meaningful contributions to the ecosystem, such as [the HexDiff project](https://diff.hex.pm/), before she was hired. In her opinion, one of Elixir's biggest assets is the community, which she considers welcoming and supportive. + +Duffel has often been able to leverage the ecosystem and reach out to existing solutions. However, they don't shy away from creating their own and open-sourcing them whenever it makes sense. Overall, the Duffel team has contributed to many areas of the ecosystem. Besides the previously mentioned Swoosh and HexDiff projects, their team members created [Hammox](https://github.com/msz/hammox), [Bigflake](https://github.com/stevedomin/bigflake), the company's own [Paginator](https://github.com/duffelhq/paginator/) library, and others. + +## Upcoming challenges + +Duffel engineers have many interesting and exciting challenges ahead of them. For example, as more developers start using the product, they will begin to hit some rate-limits imposed by airlines that they haven't yet exercised. As one would expect, different airlines have different rules and various constraints, and providing a unified solution has its hurdles. + +Some of the upcoming improvements are related to their usage of umbrella projects. Duffel started as a monolith, but they eventually migrated to Elixir's umbrella projects - a mono-repo implementation within Elixir's tooling - as soon as Phoenix v1.4 was released. Their primary motivation was to separate the communication with different airlines into different services. In the beginning, the services were clear in Steve's head, but as the team grew, they experienced friction enforcing those boundaries, which led to cyclic dependencies. + +Luckily, Elixir v1.11 started emitting warnings for cyclic and undeclared dependencies between applications, which forced the Duffel team to revisit the areas that were not strict in the past to increase the quality of the codebase in the long term. + +The team is also always exploring how to improve their APIs by bringing new approaches and technologies, such as streaming and GraphQL, as well as intelligent ways to optimize their integrations. If you are interested in tackling these and many other challenges while reshaping the travel industry, you can [learn more about Duffel's engineering opportunities](https://duffel.com/careers). diff --git a/_posts/2021-01-13-orchestrating-computer-vision-with-elixir-at-v7.markdown b/_posts/2021-01-13-orchestrating-computer-vision-with-elixir-at-v7.markdown new file mode 100644 index 000000000..8e402cae7 --- /dev/null +++ b/_posts/2021-01-13-orchestrating-computer-vision-with-elixir-at-v7.markdown @@ -0,0 +1,68 @@ +--- +layout: post +title: Orchestrating computer vision with Elixir at V7 +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at V7. +logo: /images/cases/logos/v7.png +tags: computer-vision phoenix +redirect_from: /blog/2021/01/13/orchestrating-computer-vision-with-elixir/ +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[V7](https://www.v7labs.com) is a web platform to create the sense of sight. A hub for machine learning and software engineers to develop their computer vision projects with data set management, image/video labeling, and one-click model training to automate any visual task. + +Founded in 2018 by Alberto Rizzoli and Simon Edwardsson, V7 uses Elixir, Phoenix, and Cowboy to power their web platform, responsible for managing large amounts of data and orchestrating dozens of Python nodes to carry out machine learning jobs. They have [recently closed a $3M seed round](https://www.notion.so/V7-Labs-raises-3-million-to-empower-AI-teams-with-automated-training-data-workflows-2c9b36d2043e44f3b536efae0a204632), and they are currently [hiring backend engineers to augment their Elixir team](https://www.v7labs.com/working-at-v7). + +![V7](/images/cases/bg/v7.png) + +## Visual tasks + +Throughout the years, we have been continuously automating visual tasks to speed up manual processes and reduce the rate of errors. For example: + + * Routine inspection of infrastructure: oil pipelines and offshore oil rigs require constant examination against corrosion. Once there is too much rust, it can damage the pipeline and cause leakage. Nowadays, you can use drones to take pictures and automate the detection of oxidated spots. + + * Medical examination: there is a growing use of digital pathology to assist doctors in diagnosing diseases. For example, during a biopsy of possible liver cancer, doctors use a microscope to visualize human tissue and stitch together an image of the cells, which are then individually analyzed. AI can double-check these images and help speed up problematic cells in case of positives. + + * Agriculture and farming: a wine producer may want to count grapes in a vineyard to estimate the wine production for a given season with higher precision. Farmers may use video to assess the health and the amount of exercise on free-range chickens and pigs. + + * Visual automation also plays a growing role in quality assurance and robotics: a fast-food manufacturer can use cameras to identify fries with black spots, while harvesters may use robots to pick apples from trees. + +Neural networks are at the heart of these tasks, and there is a growing need to automate the creation of the networks themselves. + +## Automating AI + +Training a neural network for image and video classification often requires multiple steps. First, you annotate images and frames with bounded-boxes, polygons, skeletons, and many other formats. The annotations are then labeled and used to train computer vision models. Labeled annotations are also used to verify models against biases, outliers, and over/underfitting. + +For many AI companies, this process exists in a loop as they continuously refine datasets and models. V7 helps teams manage and automate these steps, accelerating the creation of high-quality training data by 10-100x. Users may then export this data or use it to create neural networks directly via the platform. + + + +V7 uses Elixir to orchestrate all of these tasks. The front-end is a Vue.js application that talks to a [Phoenix-powered](https://phoenixframework.org/) API. The Phoenix application has to work with a large amount of data across a wide variety of formats. For example, a microscope outputs images in a different format, often proprietary, than a regular laboratory camera. + +To perform all the machine learning tasks, V7 has a cluster of Python nodes orchestrated by an Elixir application running the [Cowboy](https://github.com/ninenines/cowboy/) webserver. Once a Python node comes up, it establishes a WebSocket connection with Cowboy and sends how much memory, CPU, GPU, and other relevant data it has available. + +The Phoenix-powered backend communicates with the orchestrator using another Erlang VM-based technology: [RabbitMQ](https://www.rabbitmq.com/). For example, when the user tasks to auto-annotate an image, the Vue.js front-end sends a REST request to Phoenix. Phoenix then enqueues a message on RabbitMQ with the image's location, typically an Amazon S3 bucket. The orchestrator picks this message up, finds an available Python node, and delivers the relevant instructions via WebSockets. + +## Ecosystem and Infrastructure + +Other tools used by the V7 team are [Broadway](https://github.com/dashbitco/broadway) and the Erlang Distribution. + +V7 has to process and normalize images and videos. For these, they have a separate service that receives RabbitMQ messages and invokes [ImageMagick](https://imagemagick.org/) or [FFmpeg](https://ffmpeg.org/) accordingly. They use Broadway to receive RabbitMQ messages and to execute these tasks concurrently. + +The Erlang Distribution helps them broadcast information across nodes. Since they store their multimedia data on S3, they need to generate pre-signed URLs whenever the user wants to see an image or video. However, if users are routed to a different node, they would get a different URL, which would force them to download the asset again. To address this, they use the Erlang Distribution to communicate which URLs they have generated and for which purposes. + +Overall, their backend runs on Amazon ECS on about four nodes, which talk directly to PostgreSQL. The largest part of their infrastructure is the Python cluster, which takes up to two dozens of machines. + + +## Learning and Hiring + +Elixir has been present inside the company since day one, back in August 2018. Andrea Azzini, the first engineer at V7, was the one responsible for introducing it. He believed the language would be a good fit for the challenges ahead of them based on his experience running Elixir in production. + +Simon Edwardsson, their CTO, had to learn the language as they developed the system, but he was able to get up and running quickly, thanks to his previous experiences with Python and Haskell. He remarks: "As a team, we were more familiar with Django, but we were concerned it would not handle well the amount of data and annotations that we manage - which could lead to rewrites or frustrations down the road. From this perspective, the investment in Elixir was worth it, as we never had to do major changes on our backend since we started." + +Part of this is thanks to Phoenix's ability to provide high-level abstractions while making its building blocks accessible to developers: "While there is magic happening inside Phoenix, it is straight-forward to peek under the hood and make sense of everything." + +V7 has recently welcomed a new Elixir engineer to their team, making it a total of four, and they are looking for more developers interested in joining them. Historically, more engineers have applied to their machine learning positions, but they also believe many Elixir developers are prepared but don't consider themselves ready. Simon finishes with an invitation: “We are primarily looking for backend engineers with either existing Elixir experience or willingness to learn on the job. If you are interested in automating computer vision across a large range of industries, [we welcome you to get in touch](https://www.v7labs.com/working-at-v7)." diff --git a/_posts/2021-02-03-social-messaging-with-elixir-at-community.markdown b/_posts/2021-02-03-social-messaging-with-elixir-at-community.markdown new file mode 100644 index 000000000..e105822f1 --- /dev/null +++ b/_posts/2021-02-03-social-messaging-with-elixir-at-community.markdown @@ -0,0 +1,72 @@ +--- +layout: post +title: Social messaging with Elixir at Community +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Community. +logo: /images/cases/logos/community.png +tags: messaging broadway +redirect_from: /blog/2021/02/03/social-messaging-with-elixir/ +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Community](https://community.com/) is a platform that enables instant and direct communication with the people you want to reach, using the simplicity of text messaging. Used by names like Paul McCartney, Metallica, and Barack Obama, Community connects small businesses, stars, and high-profile individuals directly to their audiences. + +Community is powered by the Erlang Ecosystem, with Elixir and RabbitMQ playing central roles. This article gives an overview of the system and the tools used to handle spikes of million of users caused by events such as this tweet: + + + +## The first steps with Elixir + +Tomas Koci and Ustin Zarubin were the two engineers behind Community's initial implementation. The company was pivoting from a product they had written in Go and they felt the language was not expressive enough for the products they were building. So when faced with the challenge of developing a social messaging platform on top of SMS, they were open to trying a different stack. + +Their first encounter with Elixir was a casual one. They were chatting about the challenges ahead of them when their roommate mentioned Elixir. Shortly after, things started to click. They both had a physics background, so they found the functional paradigm quite intuitive. The Erlang VM also has its origins in telecommunications, and they were building a telecom centric product, which gave them more confidence. + +Besides the technological aspect, they also began to be active in the Elixir community. Tomas recaps: "we started attending the Elixir meetups happening here in Chattanooga. We met many developers, heard about production cases, and learned how companies like Bleacher Report were using Elixir at scale". From then on, they were sold on giving Elixir a try. + +They started their prototype in January 2018, with the intent of onboarding dozens of users. They were learning Elixir while developing the system and reaching out to potential users. + +Their first challenge was in May 2018, when one of their users announced his phone number, managed by Community, to millions of viewers. Tomas still remembers that day: "It was a Saturday night, around 11:00 pm when we saw an influx of users. It caught us by surprise and, after 10 hours, more than 400 thousand users had signed up". This influx of users stressed the system in unexpected ways, especially when it came to their upstream integrations. They had to patch the system to ensure they would not overload external systems or run over API limits they were required to conform to. + +This event also gave them insights into the types of spikes and traffic patterns the system would have to handle at scale. Early engineering hire Jeffrey Matthias urged them to break their application into different services, making it easy to scale each service individually, and he and Tomas decided to have those services communicate via message queues. + +## The next millions of users + +By October 2018, the company received funding and the newly-hired engineering team of five people, began to split the original application into services that could handle sharp increases in demand and operate at scale. Shortly after, they had their next challenge in hand: Metallica had just signed up with the platform and they were going to do an early launch with their fans on Feb 1st, 2019. + +The team is glad to report the announcement was a success with no hiccups on their end. They were then five backend engineers who tackled everything from architectural design and development to setting up and operating the whole infrastructure. + +Community was officially unveiled in May 2019, [attracting hundreds of music stars shortly after](https://www.billboard.com/amp/articles/business/8543190/why-hundreds-music-stars-giving-fans-phone-numbers-community-app). Fourteen months later, [Barack Obama tweeted to millions his phone number powered by Community](https://twitter.com/barackobama/status/1308769164190941187). + +## The current architecture + +Today, more than 60 services with distinct responsibilities power Community, such as: + +* A message hub between community leaders and members +* User data management +* Media services (video, audio, images) +* Systems for Community's internal team +* Data science and machine learning +* Billing, administration, etc + +The vast majority of those services run Elixir, with Python covering the data science and machine learning endpoints, and Go on the infrastructure side. + +[RabbitMQ](https://www.rabbitmq.com/) handles the communication between services. The Erlang-backed message queue is responsible for broadcasting messages and acting as [their RPC backbone](https://andrealeopardi.com/posts/rpc-over-rabbitmq-with-elixir/). Messages between services are encoded with Protocol Buffers via [the protobuf-elixir library](https://github.com/elixir-protobuf/protobuf). + +Initially, they used [the GenStage library](http://github.com/elixir-lang/gen_stage/) to interface with RabbitMQ, but they have migrated to the higher level [Broadway](https://github.com/dashbitco/broadway) library over the last year. Andrea Leopardi, one of their engineers, outlines their challenges: "Our system has to handle different traffic patterns when receiving and delivering data. Incoming data may arrive at any time and be prone to spikes caused by specific events powered by actions within Communities. On the other hand, we deliver SMSes in coordination with partners who impose different restrictions on volumes, rate limiting, etc." + +He continues: "both GenStage and Broadway have been essential in providing abstractions to handle these requirements. They provide back-pressure, ensure that spikes never overload the system, and guarantee we never send more messages than the amount defined by our delivery partners". As they implemented the same patterns over and over in different services, they found Broadway to provide the ideal abstraction level for them. + +Their most in-demand service, the message hub, is powered by only five machines. They use [Apache Mesos](https://mesos.apache.org/) to coordinate deployments. + +## Growing the team + +Community's engineering team has seen stable growth over the last two years. Today they are 25 backend engineers, the majority being Elixir devs, and the company extends beyond 120 employees. + +Karl Matthias, who joined early on, believes the challenges they face and the excitement for working on a new language has been positive for hiring talent. He details: "we try to hire the best production engineers we can, sometimes they know Elixir, sometimes they don't. Our team has generally seen learning Elixir as a positive and exciting experience". + +The team is also happy and confident about the stability Elixir provides. Karl adds: "Elixir supervisors have our back every time something goes wrong. They automatically reestablish connections to RabbitMQ, they handle dropped database connections, etc. The system has never gone wrong to the point our infrastructure layer had to kick-in, which has been quite refreshing." + +The Community team ended our conversation with a curious remark. They had just shut down their first implementation of the system, the one that received a sudden spike of four hundred thousand users on a Saturday night. Tomas concludes: "it is pretty amazing that the service we implemented while learning Elixir has been running and operating in production just fine, even after all of these milestones. And that's generally true for all of our services: once deployed, we can mostly forget about them". diff --git a/_posts/2021-04-02-marketing-and-sales-intelligence-with-elixir-at-pepsico.markdown b/_posts/2021-04-02-marketing-and-sales-intelligence-with-elixir-at-pepsico.markdown new file mode 100644 index 000000000..dc81c1aa8 --- /dev/null +++ b/_posts/2021-04-02-marketing-and-sales-intelligence-with-elixir-at-pepsico.markdown @@ -0,0 +1,56 @@ +--- +layout: post +title: Marketing and sales intelligence with Elixir at PepsiCo +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at PepsiCo. +logo: /images/cases/logos/pepsico.png +tags: biz-intelligence phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +PepsiCo is one of the world's leading food and beverage companies serving more than 200 countries and territories around the world. Today Elixir is used at varying capacities inside PepsiCo by six different teams. This article explores how the Search Marketing and Sales Intelligence Platform teams adopted and use Elixir to build internal tools. + +Although we will explore only two teams in this article, [PepsiCo is hiring Elixir engineers across multiple teams](https://www.pepsicojobs.com/teams-ecommerce). Let's get started. + +![PepsiCo](/images/cases/bg/pepsico.jpg) + +## The first steps + +The project that would become the first Elixir project and open the door for future Elixir applications inside PepsiCo was started by Jason Fertel back in 2016. + +Initially, the application provided workflow automation for managing search marketing operations on multiple web platforms. The product was a success and ended up integrated into PepsiCo in 2018. + +Now, the Elixir application plays a central role in a data pipeline that empowers PepsiCo's marketing and sales teams with tools to query, analyze, and integrate with several search marketing partners. + +The pipeline starts with the Data Engineering team, which collects and stores data into [Snowflake Data Cloud](https://www.snowflake.com/). The Elixir application reads data from Snowflake’s platform, pre-process, and stores it in two databases: [PostgreSQL](https://www.postgresql.org/) or [Apache Druid](https://druid.apache.org/), according to the data characteristics. Finally, a Phoenix application serves this data to internal teams and communicates directly with third-party APIs. + +## Why Elixir? + +Elixir helps PepsiCo eCommerce focus and get things done fast. “Elixir allows our team to develop quickly with confidence,” says David Antaramian, a Software Engineering Manager at PepsiCo. “In turn, that lets us deliver value to the business quickly, and it’s the reason we’ve stuck with the language. Whether it’s streaming changes to the front-end or orchestrating concurrent data operations across multiple storage systems, Elixir offers a robust developer experience that translates to a great consumer experience.” + +Different Elixir features came together to help the PepsiCo team build compelling development and user experiences. Thanks to its functional and extensible aspects, PepsiCo used Elixir to create a domain-specific language that translates business queries into data structures sent to different stores. This gave them a stable foundation where they can continually add new queries and integrations, even as they grow in complexity. + +Furthermore, the reports generated by PepsiCo's marketing and sales teams often have to query different tables or even separate storages, all while juggling long-running connections to different third-party APIs. Elixir's programming model, inherited from the Erlang Virtual Machine, makes it trivial to run all of these operations concurrently, leading to fast and rich user interactions while the development team remains focused on delivering features. + +## Libraries and frameworks + +David Antaramian is quick to praise the Erlang runtime and its standard library. He says: "Since we are working with large amounts of data, it is also essential to avoid hitting the database whenever possible. Thankfully, Erlang ships with an in-memory table storage called [ETS](http://www.erlang.org/doc/man/ets.html), which we use to store hundreds of thousands of rows". + +The Erlang standard library was also handy when communicating to some data stores. In particular, the Snowflake platform requires ODBC connections. The PepsiCo team built a library called [Snowflex](https://github.com/pepsico-ecommerce/snowflex), designed on top of [Erlang's built-in ODBC drivers](http://www.erlang.org/doc/man/odbc.html). + +The Elixir ecosystem nicely complements the Erlang one. The front-end, written in React, talks to the server via [the Absinthe GraphQL toolkit](http://absinthe-graphql.org/) running on top of [the Phoenix web framework](http://phoenixframework.org/). The [Ecto database library](https://github.com/elixir-ecto/ecto) manages the communication to PostgreSQL. They also use the [esaml](https://github.com/handnot2/esaml) and [Samly](https://github.com/handnot2/samly) libraries to provide authentication within PepsiCo's organization - another example of leveraging the tools within both Erlang and Elixir communities. + +Finally, the team also recognizes the [Erlang Ecosystem Foundation](https://erlef.org/)'s efforts, which PepsiCo are sponsors of, particularly the Observability Working Group. David remarks: "The adoption of Telemetry by the ecosystem has been a massive help in bringing monitoring visibility and metrics to our system. Now when we see spikes in one place, we can easily correlate them with other system parts". + +## Hiring + +Today there are approximately 40+ Elixir engineers within PepsiCo distributed across six teams. Eight of those engineers are part of the Search Marketing and Sales Intelligence Platform teams. + +While the team recognizes that there aren't as many Elixir engineers compared to communities like JavaScript, they were effective in hiring qualified Elixir candidates. Chase Gilliam, a Software Engineering Manager at PepsiCo, explains: "We have met many engineers that, like us, found Elixir due to being burned out by previous experiences. So when it came to hiring, many Elixir candidates had a mindset similar to ours, which ultimately sped up the process." + +This initial group of Elixir engineers paved the way for the language's growth inside PepsiCo. David adds: "At first, we looked for engineers with Elixir experience to help us build a team that could guide other developers. Then we extended the pool to anyone who has a functional programming background and now to developers with either Ruby or Erlang experience. However, if someone is the right candidate, we onboard them even if they have no Elixir experience and train them". He continues: "We also make extensive use of the learning resources available in the community, such as conferences, books, online courses, and others." + +As the team grew, they adopted best practices and saw the quality of the codebase improve at the same time. Chase concludes: "At the beginning, there were some large modules in our application. Luckily, refactoring in a functional programming language is straightforward, thanks to immutability and limited side-effects. Adopting tools like Credo, ExDoc, and the code formatter was also essential to standardize how we use Elixir internally." For those interested in learning more about the different use cases for Elixir inside PepsiCo and help continue its growth, [they are hiring](https://www.pepsicojobs.com/teams-ecommerce). diff --git a/_posts/2021-05-19-elixir-v1-12-0-released.markdown b/_posts/2021-05-19-elixir-v1-12-0-released.markdown new file mode 100644 index 000000000..676f3d2e2 --- /dev/null +++ b/_posts/2021-05-19-elixir-v1-12-0-released.markdown @@ -0,0 +1,102 @@ +--- +layout: post +title: Elixir v1.12 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.12 is out with improvements to scripting, tighter Erlang/OTP 24 integration, stepped ranges, and dozen of new functions across the standard library +--- + +Elixir v1.12 is out with improvements to scripting, tighter Erlang/OTP 24 integration, stepped ranges, and dozen of new functions across the standard library. Overall this is a small release, which continues our tradition of bringing Elixir developers quality of life improvements every 6 months. Some of these improvements directly relates with the [recent efforts of bringing Numerical Computing to Elixir](https://dashbit.co/blog/nx-numerical-elixir-is-now-publicly-available). + +Elixir v1.12 requires Erlang/OTP 22+. We also recommend running `mix local.rebar` after installation to upgrade to the latest Rebar versions, which includes support for Erlang OTP/24+. + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, `noscript` tags with the proper links will be shown. + +## Scripting improvements: `Mix.install/2` and `System.trap_signal/3` + +Elixir v1.12 brings new conveniences for those using Elixir for scripting (via `.exs` files). Elixir has been capable of managing dependencies for a quite long time, but it could only be done within Mix projects. In particular, the Elixir team is wary of global dependencies as any scripts that rely on system packages are brittle and hard to reproduce whenever your system changes. + +`Mix.install/2` is meant to be a sweet spot between single-file scripts and full-blown Mix projects. With `Mix.install/2`, you can list your dependencies at the top of your scripts. When you execute the script for the first time, Elixir will download, compile, and cache your dependencies before running your script. Future invocations of the script will simply read the compiled artifacts from the cache: + +```elixir +Mix.install([:jason]) +IO.puts(Jason.encode!(%{hello: :world})) +``` + +`Mix.install/2` also performs protocol consolidation, which gives script developers an option to execute their code in the most performant format possible. Note `Mix.install/2` is currently experimental and it may change in future releases. + +Furthermore, `Mix.install/2` pairs nicely with Livebook, a newly announced project that brings interactive and collaborative notebook projects to Elixir. With Livebook and `Mix.install/2`, you can bring dependencies into your code notebooks and ensure they are fully reproducible. [Watch the Livebook announcement to learn more](https://www.youtube.com/watch?v=RKvqc-UEe34). + +Another improvement to scripting is the ability to trap exit signals via `System.trap_signal/3`. All you need is the signal name and a callback that will be invoked when the signal triggers. For example, ExUnit leverages this functionality to print all currently running tests when you abort the test suite via SIGQUIT (`Ctrl+\\ `). You can see this in action when running tests in the Plug project below: + + + +This is particularly useful when your tests get stuck and you want to know which one is the culprit. + +**Important**: Trapping signals may have strong implications on how a system shuts down and behaves in production and therefore it is extremely discouraged for libraries to set their own traps. Instead, they should redirect users to configure them themselves. The only cases where it is acceptable for libraries to set their own traps is when using Elixir in script mode, such as in `.exs` files and via Mix tasks. + +## Tighter Erlang/OTP 24 integration + +[Erlang/OTP 24 ships with JIT compilation](https://blog.erlang.org/My-OTP-24-Highlights/) and Elixir developers don't have to do anything to reap its benefits. There are many other features in Erlang/OTP 24 to look forwards to and Elixir v1.12 provides integration with many of them: such as support for 16bit floats in bitstrings as well as performance improvements in the compiler and during code evaluation. + +Another excellent feature in Erlang/OTP 24 is the implementation of [EEP 54](http://www.erlang.org/eeps/eep-0054.html), which provides extended error information for many functions in Erlang's stdlib. Elixir v1.12 fully leverages this feature to improve reporting for errors coming from Erlang. For example, in earlier OTP versions, inserting an invalid argument into an ETS table that no longer exists would simply error with `ArgumentError`: + + + +However, in Elixir v1.12 with Erlang/OTP 24: + + + +Finally, note Rebar v2 no longer works on Erlang/OTP 24+. Mix defaults to Rebar v3 since Elixir v1.4, so no changes should be necessary by the vast majority of developers. However, if you are explicitly setting `manager: :rebar` in your dependency, you want to move to Rebar v3 by removing the `:manager` option. Compatibility with unsupported Rebar versions will be removed from Mix in the future. + +## Stepped ranges + +Elixir has had support for ranges from before its v1.0 release. Ranges support only integers and are inclusive, using the mathematic notation `a..b`. Ranges in Elixir are either increasing `1..10` or decreasing `10..1` and the direction of the range was always inferred from the first and last positions. Ranges are always lazy as its values are emitted as they are enumerated rather than being computed upfront. + +Unfortunately, due to this inference, it is not possible to have empty ranges. For example, if you want to create a list of `n` elements, you cannot express it with a range from `1..n`, as `1..0` (for `n=0`) is a decreasing range with two elements. + +Elixir v1.12 supports stepped ranges via [the `first..last//step` notation](https://hexdocs.pm/elixir/1.12/Kernel.html#..///3). For example: `1..10//2` will emit the numbers `1`, `3`, `5`, `7`, and `9`. You can consider the `//` operator as an equivalent to "range division", as it effectively divides the number of elements in the range by `step`, rounding up on inexact scenarios. Steps can be either positive (increasing ranges) or negative (decreasing ranges). Stepped ranges bring more expressive power to Elixir ranges and they elegantly solve the empty range problem, as they allow the direction of the steps to be explicitly declared instead of inferred. + +As of Elixir v1.12, implicitly decreasing ranges are soft-deprecated and warnings will be emitted in future Elixir versions based on our [deprecation policy](https://hexdocs.pm/elixir/compatibility-and-deprecations.html#deprecations). + +## `then/2` and `tap/2` + +Two new functions have been added to `Kernel` module, in order to ease working with pipelines. [`tap/2`](https://hexdocs.pm/elixir/1.12/Kernel.html#tap/2) passes the given argument to an anonymous function, returning the argument itself. [`then/2`](https://hexdocs.pm/elixir/1.12/Kernel.html#then/2) passes the given argument to an anonymous function, returning the result. The following: + +```elixir +"hello world" +|> tap(&IO.puts/1) +|> then(&Regex.scan(~r/\w+/, &1)) +``` + +Is equivalent to this: + +```elixir +"hello world" +|> (fn x -> + IO.puts(x) + x + end).() +|> (&Regex.scan(~r/\w+/, &1)).() +``` + +Both `tap/2` and `then/2` are implemented as macros, and compiler improvements available on Erlang/OTP 24 ensure the intermediate anonymous functions is optimized away, which guarantees the idioms above do not have any performance impact on your code. + +## IEx improvements + +IEx got two important quality of life improvements in this release. Hitting tab after a function invocation will show all of the arguments for said function and it is now possible to paste code with pipelines in the shell. See both features in action below: + + + +## Additional functions + +Elixir v1.12 has also added many functions across the standard library. The `Enum` module received additions such as `Enum.count_until/2`, `Enum.product/1`, `Enum.zip_with/2`, and more. The `Integer` module now includes `Integer.pow/2` and `Integer.extended_gcd/2`. + +The `Code` module got a [`cursor_context/2`](https://hexdocs.pm/elixir/1.12/Code.html#cursor_context/2) function, which is now used to power `IEx` autocompletion and it is [used by projects such as Livebook to provide intellisense](https://user-images.githubusercontent.com/17034772/115117125-533b2900-9f9d-11eb-94a9-a2cf2ccb7388.mp4). + +The EEx application has also been extended to provide metadata on text segments. This has enabled the Surface and Phoenix LiveView teams to implement [a new template language called HEEx](https://github.com/phoenixframework/phoenix_live_view/pull/1440), which validates both HTML and EEx. Finally, the `Registry` module supports the `:compressed` option, which is useful for GraphQL applications managing hundreds of thousands of subscriptions via [Absinthe](http://absinthe-graphql.org/). + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.12.0). Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2021-06-02-social-virtual-spaces-with-elixir-at-mozilla.markdown b/_posts/2021-06-02-social-virtual-spaces-with-elixir-at-mozilla.markdown new file mode 100644 index 000000000..51244a2e2 --- /dev/null +++ b/_posts/2021-06-02-social-virtual-spaces-with-elixir-at-mozilla.markdown @@ -0,0 +1,54 @@ +--- +layout: post +title: Social virtual spaces with Elixir at Mozilla +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Mozilla. +logo: /images/cases/logos/mozilla-hubs.png +tags: virtual-spaces phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Hubs](https://hubs.mozilla.com/) is Mozilla's take on virtual social experiences. You build your own private spaces and share them with your friends, co-workers, and community. Avatars in this space can move freely in a 3D social environment and watch videos, exchange messages, and talk to other people nearby. All you need is a browser and a microphone! + +Hubs is [fully](https://github.com/mozilla/hubs) [open source](https://github.com/mozilla/reticulum) and you can host it on your infrastructure via [Hubs Cloud](https://hubs.mozilla.com/cloud). Community managers, educators, and event organizers have been using Hubs Cloud to run virtual events and online activities tailored to their specific brandings and needs. All it takes to run your own version of Hubs is one click away - which perhaps makes Hubs the most deployed Phoenix application ever! + +![Mozilla Hubs](/images/cases/bg/mozilla-hubs.jpg) + +## From VR to Elixir + +The Hubs team started at Mozilla as the Mixed Reality team about 3.5 years ago. Their main goal was to explore ways for online social interaction via avatars and mixed reality. + +They quickly focused on building their first proof of concept, where avatars could communicate, move around, and join different rooms, everything running directly in the browser. This was a significant departure from the state of the art of Virtual Reality everywhere, as the getting started experience up to this point was cumbersome and often required steep investment in the form of VR headsets. + +The initial prototype was a success and it pushed the team to build a product. However, all communication in the proof of concept was peer-to-peer, which limited the features and experiences they could provide. Therefore the Hubs team knew they needed a capable backend technology to provide fan-out communication and coordinate how all different avatars interact within the virtual spaces. John Shaughnessy, Staff Software Engineer at Mozilla, comments: "When you get a lot of people in the same space, be it virtual or in the real world, there is never only a single conversation going on. Once you get ten or twenty different people in a room, conference calls don’t work well. In Hubs, people transition between multiple simultaneous conversations simply by moving around". + +With this bold vision in hand, they assessed their whole stack. They settled on using JavaScript with [Three.js](https://threejs.org/) in the front-end and chose [the Phoenix web framework](https://phoenixframework.org/) for the backend. Greg Fodor, who was an Engineering Manager at Mozilla at the time, explains the choice: "We first listed all of the features we had to implement, from trivial things like REST endpoints, to more complex use cases, such as chat messaging and tracking where avatars are in the virtual world. Once I started to learn Phoenix, I saw all of those features were already there! The application we were building has to manage a large number of connections with real-time low latencies, something we knew the Erlang VM was an excellent fit for". + +## In production + +Hubs went live in January 2018. Almost everything in Hubs goes through Phoenix. The only exception is the WebRTC voice channels, which are handled by designated voice servers, initially implemented with [Janus](https://janus.conf.meetecho.com/) and later ported to [MediaSoup](https://mediasoup.org/). However, the Phoenix app still manages the voice servers and how connections are assigned to them. + +The deployment is orchestrated by [Habitat](https://www.chef.io/products/chef-habitat/) and running on Amazon EC2. Habitat provides packaging and orchestration. When a voice server joins the Habitat ring, the Phoenix services receive a message and start assigning voices to voice servers. Overall they run on 4 Phoenix and 4 voice servers. + +The Elixir experience in production has been quite smooth. Dominick D'Aniello, Staff Software Engineer at Mozilla, points out some areas they discussed improving: "the Phoenix application works mostly as a proxy, so we avoid decoding and reencoding the data unless we really need to. But sometimes we have to peek at the payloads and JSON is not the most efficient format to do so." They have also considered relying more on Elixir processes and the Erlang distribution. Dominick continues: "when a new client joins, it needs to ask all other clients what is their state in the world, what they own, and what they care about. One option is to use Elixir processes in a cluster to hold the state of the different entities and objects in the virtual world". + +## Beyond Hubs + +With many large companies investing in online communication, the Mozilla team saw the possibility of virtual spaces becoming walled-gardens inside existing social platforms. This led the Hubs team to work on Hubs Cloud, with the mission to commoditize virtual spaces by allowing anyone to run their own version of Hubs with a single click. + +Hubs Cloud launched in February 2020 and it has been a hit. [New York University did its graduation ceremony on a Hubs Cloud instance](https://twitter.com/nyuniversity/status/1258401916096315399). [The IEEE Virtual Reality Conference embraced Hubs](https://www.computer.org/conferences/organize-a-conference/organizer-resources/hosting-a-virtual-event/success-stories/IEEE-VR-2020) for a more accessible and sustainable event with talks and poster sessions all happening in virtual rooms, while [the Minnesota Twins baseball team launched a Virtual Hall of Fame](https://www.twincities.com/2021/02/09/twins-set-to-launch-new-virtual-fan-experience/) on top of the platform. + +Their cloud version uses Amazon CloudFormation to instantiate Hubs inside the user's account. This approach brought different challenges to the Hubs team: "we want Hubs Cloud to be as affordable and straightforward as possible. The Phoenix app has already been a massive help on this front. We have also moved some features to Amazon Lambda and made them optional, such as image resizing and video conversion" - details John. + +Since Hubs is also open source, developers can run their own Hubs instance in whatever platform they choose or change it however they want. That's the path Greg Fodor recently took when he announced [Jel](https://jel.app/): "Jel is the video game for work. It is a mashup of Minecraft and Discord, where everything is 3D. My goal is to spark new directions and ideas to get people excited about VR". + +## Summing up + +Today, the Hubs team has 10 contributors, half of whom are developers. Their engineering team is quite general and learning Elixir happens organically: "you are motivated by the feature you are working on. If it requires changing the backend, you learn Elixir with the help of the team and then make your contribution". + +Overall, the bet on Phoenix was a successful one. Greg Fodor highlights: "The most significant benefit of Phoenix is in using a stack that excels at solving a large variety of problems. Once onboarded to Phoenix, there is a huge surface area our engineers can touch. Any feature they come up with, they can run with it. And because Hubs is open source, our contributors will also have the same experience. Overall, Elixir and Phoenix reduce the effort needed to cause the largest impact possible across our whole product". + +Lately, they have leaned even further into the ecosystem, as they have started exposing Hubs APIs over GraphQL with the help of Absinthe. They have also migrated to Phoenix v1.5 and are using the [Phoenix LiveDashboard](https://github.com/phoenixframework/phoenix_live_dashboard) to provide metrics and instrumentation to Hubs Cloud users. diff --git a/_posts/2021-07-29-bootstraping-a-multiplayer-server-with-elixir-at-x-plane.markdown b/_posts/2021-07-29-bootstraping-a-multiplayer-server-with-elixir-at-x-plane.markdown new file mode 100644 index 000000000..e0563549c --- /dev/null +++ b/_posts/2021-07-29-bootstraping-a-multiplayer-server-with-elixir-at-x-plane.markdown @@ -0,0 +1,56 @@ +--- +layout: post +title: Bootstrapping a multiplayer server with Elixir at X-Plane +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at X-Plane. +logo: /images/cases/logos/x-plane.png +tags: multiplayer udp otp +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[X-Plane 11](https://www.x-plane.com/) is the world's most comprehensive and powerful flight simulator for personal computers and mobile devices. X-Plane is not a game but an engineering tool created by Laminar Research that can be used to predict the flying qualities of fixed- and rotary-wing aircraft with incredible accuracy. The X-Plane franchise comes in both consumer and FAA-certifiable professional versions. + +Recently, the X-Plane team took on the challenge of adding a multiplayer experience with the goal of hosting north of 10000 users in the same session. This article explores why they chose Elixir and how a team of one developer - without prior language experience - learned the language and deployed a well-received multiplayer experience in 6 months. The overall solution features a brand new open-source implementation of the RakNet communication protocol in Elixir and overperforms the original requirements when put under unexpected load. + +![X-Plane](/images/cases/bg/x-plane.jpg) + +## Requirements + +The X-Plane team has offered peer-to-peer multiplayer in the simulator for a long time but never server-hosted multiplayer. This was a new journey for them and they had complete freedom to pick the technology stack. [According to their blog post](https://developer.x-plane.com/2021/01/have-you-heard-the-good-news-about-elixir/), their goals were: + +1. To build a rock-solid server with error isolation. For example, an exception during a client update should not bring the whole server down. + +2. To implement a single shared world that can scale to tens of thousands of concurrent pilots. + +3. To iterate quickly: because this was the first time the Laminar Research team offered a hosted multiplayer environment, they wanted to move quickly to ship this system. This would allow users to begin flying in groups immediately and serve as a platform to gauge interest in further feature development. + +4. To be fast and consistent. Multiplayer has a "soft real-time" constraint, and they need to service _all_ clients consistently and on time. Quantitatively, this means the 99th percentile response times matter a lot more than the mean or median. + +From those requirements, the need for stability and fast iteration ruled out low-level languages, even the ones in which they had plenty of in-house experience. + +The need for speed and vertical scalability excluded many modern web languages, such as Ruby and Python, where the model for scaling up is generally throwing more servers at it. It was essential to avoid synchronizing state across multiple machines, which requires more development time and worsens the user experience due to the increased latency. + +They eventually settled on three top contenders: Rust, Go, and Elixir. Elixir took the edge thanks to two exclusive features: fault tolerance and predictable latency. Both are built into the very core of the Erlang Virtual Machine - the robust platform that Elixir runs on. Tyler Young, X-Plane's engineer leading this implementation, highlights: "We wanted a stack that could max server capacity. We would rather run a 64-core machine than dozens of 4-core VMs. Saša Jurić's talk, [the Soul of Erlang and Elixir](https://www.youtube.com/watch?v=JvBT4XBdoUE), showed us that the concurrency model, process isolation, and partial restarts provided by the platform were the abstractions we were looking for." + +## Modeling multiplayer with Elixir + +Ready to give Elixir a try, Tyler picked up a couple books but soon realized the language's [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) provided the background he needed. He explains: "while the introductory guide covers the language constructs, the advanced guide on the website has you build an actual project with TCP connections, with the basic architectural patterns we would use in production." + +However, instead of jumping headfirst into the multiplayer server, he decided to give Elixir a try on a smaller problem. He wrote a web proxy to the National Oceanic and Atmospheric Administration (NOAA) weather services and put it in production. This experience taught him the importance of leveraging all of the instrumentation and metrics provided by the Erlang VM. They chose [AppSignal](https://www.appsignal.com/) to help consume and digest this information. + +Two weeks later, he started working on the server by implementing [the UDP-centric RakNet protocol in Elixir](https://en.wikipedia.org/wiki/RakNet). Unfortunately, there is little documentation, so they had to refer to the reference implementation in C++ most of the time. Luckily, thanks to its roots in telecommunication and network services, [Elixir and Erlang have built-in support for parsing binary packets](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#%3C%3C%3E%3E/1), which made the task a joy. The team also mapped each UDP connection to distinct lightweight threads of execution in Elixir, which we call _processes_. Elixir processes are cheap, isolated, concurrent, and are fairly scheduled by the runtime. This design allowed the X-Plane team to fully leverage the properties of robustness and predictable latency that first attracted them to the platform. Their implementation is written on top of Erlang's [gen_udp](http://www.erlang.org/doc/man/gen_udp.html) and [is open source](https://github.com/X-Plane/elixir-raknet). + +Five months after choosing Elixir, they began welcoming beta testers into the server. The community's reaction was overwhelmingly positive, and the new multiplayer experience led to a strong uptick in the number of subscriptions as it went live a month later. + +## Deployment and keeping it simple + +At the moment, X-Plane's player base in North America is powered by a single server, running on 1 eight-core machine with 16GB of memory, although only 200MB or 300MB of memory is effectively used. Each connected player sends 10 updates a second. + +For deployments, they use a blue-green strategy, alternating between two servers of the same capacity. Tyler explains: "We are aware the Erlang VM provides hot code swapping and distribution, but we are taking the simplest route whenever possible. It is much easier for us to alternate between two servers during deployments, as the servers are stable and we don't deploy frequently. Similarly, when it comes to distribution, we prefer to scale vertically or set up different servers in different regions for players across the globe." + +Paul McCarty, who joined the project after launch, can attest to its simplicity: "even without prior Elixir experience, I was able to jump in and add new functionality to our HTTP APIs early on." Those APIs are built on top of [Plug](http://github.com/elixir-lang/plug) to power their chat services, provide information about connected users, and more. He concludes: "When adding new features, the server development is never the bottleneck." + +Paul and Tyler finished our conversation with a curious anecdote: a couple months ago, they distributed an updated client version with debug code in it. This additional code caused each connected user to constantly ping the server every 100ms, even if not in multiplayer mode. This caused their traffic to increase 1000x! They only discovered this increase 2 weeks later when they saw the CPU usage in their Elixir server went from 5% to 21%. Once they found out the root cause and how the system handled it, they realized they didn't have to rush a client update to remove the debug code and they chose to maintain their regular release cycle. At the end of the day, it was a perfect example of the confidence they gained and built around the language and platform. diff --git a/_posts/2021-11-10-embracing-open-data-with-elixir-at-the-ministry-of-ecological-transition-in-france.markdown b/_posts/2021-11-10-embracing-open-data-with-elixir-at-the-ministry-of-ecological-transition-in-france.markdown new file mode 100644 index 000000000..7326c7d57 --- /dev/null +++ b/_posts/2021-11-10-embracing-open-data-with-elixir-at-the-ministry-of-ecological-transition-in-france.markdown @@ -0,0 +1,70 @@ +--- +layout: post +title: Embracing open data with Elixir at the Ministry of Ecological Transition in France +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at the Ministry of Ecological Transition in France. +logo: /images/cases/logos/met-france.svg +tags: open-data gov phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +A group of initiatives towards innovation and open data has given the opportunity for Elixir to play a central role in exploring, validating, and visualizing transportation data across all of France. This article will show how Elixir came to power [the National Access Point for transport data in France](https://transport.data.gouv.fr/) and explore why it continues to be an excellent fit thanks to its real-time libraries, educational tooling, and orchestration capabilities. + +![Sample map of transports](/images/cases/bg/met-france.png) + +## State Startups + +In 2013, the French Government launched a Digital Services incubator, called [beta.gouv.fr](https://beta.gouv.fr), to spread the culture of digital innovation throughout the administration. They do this through State Startups. + +State Startups are a match between a team and a mission. They help "intrapreneurs" - public servants who identify frictions and opportunities to improve the lives of their fellow citizens - tackle real-world challenges alongside a team of experts. This team of 2 to 4 people has six months to build a proof of concept ([see official website](https://beta.gouv.fr/en/)). + +The insight is: instead of trying to convince other stakeholders to work towards specific goals, it is better to empower innovation and those who want to drive change. Those individuals are given a budget and the autonomy to choose the technology and assemble their team. In exchange, they must open-source and publicly document all code, costs, metrics, and statistics. + +The first State Startup was [data.gouv.fr](https://www.data.gouv.fr/en), which transformed France's open data efforts from a catalog of spreadsheets into a social platform that closed the gap between the citizens consuming the data and the institutions providing them. The tool is fully [open-source](https://github.com/opendatateam/udata), allowing other countries to use it in production too. + +At the time of writing, [261 State Startups](https://beta.gouv.fr/startups/) have been launched and are in various states of development. + +## Elixir drives by + +In 2017, a team was assembled to begin a new State Startup focused on transportation data. An incoming European delegated regulation would make it mandatory for institutions and corporations to make transportation data public. The State Startup aimed at preparing the ecosystem actors for this regulatory change. + +To address this, the team decided to build a web application to search and visualize the existing transportation data available in [data.gouv.fr](https://www.data.gouv.fr). They initially targeted public transportation information provided by cities about buses, subways, and trams, all available in a static format called General Transit Feed Specification ([GTFS](https://gtfs.org/)) ([live example](https://transport.data.gouv.fr/resources/50471#visualization)). + +The two developers of the team, Vincent Lara and Mauko Quiroga, had heard about Elixir and were interested in learning more. They understood it could provide a robust but flexible and fun platform to explore the problem space. So [they bootstrapped the application](https://github.com/etalab/transport-site/commit/837a048c37ac31151b51ac09432dbcbff3917de5) with the [Phoenix web framework](https://phoenixframework.org/). + +As they developed the system, they spotted gaps and errors in the data available. So they began validating the data and reaching out to the institutions publishing them, providing direct feedback and value to the open data platform. The incubator saw the benefits from their contributions and, after a few months, they had a successful State Startup in their hands alongside the approval to continue working on their mission. + +Between 2017 and 2021, the multi-disciplinary team (one public servant, “business developers”, and technical staff) worked to increase the coverage of published transportation data and helped cities and operators to reach their technical and regulatory goals. + +## Current challenges + +In 2021, the State Startup has "graduated" from its “beta gouv” incubator and is now part of France's Ministry of Ecological Transition. Now composed by Francis Chabouis, Thibaut Barrère, and Antoine Augusti, the technical part of the team is tackling new use cases and challenges as the platform grows in terms of needs and versatility. + +Many of those are driven by the adoption of new data formats by governments and corporations. For example, [GTFS](https://github.com/google/transit/tree/master/gtfs/spec/en) provides a static (theoretical) itinerary: if a bus is currently delayed, this information would not be available in the feed. Enter the [GTFS-RT](https://github.com/google/transit/tree/master/gtfs-realtime/spec/en) format, where RT stands for real-time, to address those gaps. The General Bikeshare Feed Specification ([GBFS](https://nabsa.net/resources/gbfs/)) ([live example](https://transport.data.gouv.fr/datasets/velos-libre-service-creteil-cristolib-disponibilite-en-temps-reel/)) tracks bicycles, scooters, carpooling, etc. Plus the [SIRI](https://en.wikipedia.org/wiki/Service_Interface_for_Real_Time_Information) (Service Interface for Real-time Information) and [NeTEx](https://en.wikipedia.org/wiki/NeTEx) families of protocols. + +Some of those formats have supporting technologies (validators, converters) written in other languages (Java, Rust, etc), which would be beneficial to integrate with. The team then realized the way forward is to adapt their Elixir system to orchestrate and coordinate those subsystems. Luckily, Elixir has shown to be exceptionally well suited to this task, thanks to the underlying Erlang Virtual Machine, designed for communication systems. Francis Chabouis expands: "We currently need to integrate with internal and external services in a variety of formats. Some are static, some require periodic pulls, and others keep open connections to push data. Elixir allows us to experiment, iterate, and scale those ideas quickly". + +Overall, the data catalog now includes: + +* Timetables for buses, subways, and trains, including trips and operators, as real-time updates +* Bicycle lanes and carpooling areas +* Charging and refueling stations +* Private parking areas +* Location of rental cars, scooters, bicycles, and others + +Many of those formats also bring real-time concerns as they evolve the application to sample and show events as they happen. This is where the team is currently working at leveraging [Phoenix LiveView](http://github.com/phoenixframework/phoenix_live_view) to build the interactivity and functionality they need while keeping their stack simple and productive. + +The technical team has also recently grown to three developers, totaling seven members altogether. To prepare themselves for the new team members, Thibaut Barrère was responsible for upgrading their dependencies, including Elixir and Erlang, which were largely unchanged from 2017. While performing such changes can often be daunting in other stacks, Thibaut shares a very positive experience: "we did not see any breaking changes after performing long-due updates. Overall, the language and libraries seem quite stable and careful to avoid breaking changes. This experience gives us the confidence we can continue focusing on the needs of our users as we move forward". + +## Open data and education + +As with any other team, some challenges go beyond the technical aspects. For example, they sometimes spot companies and cities that do not make their transportation data available, which is against European Law. The team often heard concerns about making parts of their systems public, which could lead to failures in critical infrastructure. + +To address this, the team built [a small Phoenix application](https://github.com/etalab/transport-site/tree/master/apps/unlock) that works as a simple proxy to those systems. The proxy caches the data for specific intervals, helping those entities address the security and traffic concerns around their critical systems. The application uses [Cachex](https://github.com/whitfin/cachex) and provides a real-time dashboard, built with LiveView, where they can configure the system, track application load, and see cache usage data. + +Another area the team is actively investigating is how to make the data itself more accessible to developers who want to consume it. A non-trivial amount of work is required between fetching the city data, parsing it, and displaying it on a map, which can discourage developers from beginning their open data journey. To this end, they plan to assemble a collection of [Livebooks](http://github.com/livebook-dev/livebook), a recently released tool for writing code notebooks in Elixir that allows developers to get up and running quickly and obtain immediate feedback on their code. + +Thibaut remarks how the growth of the language and its ecosystem supports their application and needs: "every time we faced a new challenge, a solution was readily available to us. When we needed to orchestrate multiple subsystems, the stack excelled at it. When we required real-time features, Phoenix and LiveView had first-class support for it. Now we need to promote education and access to open data, and Livebook is shaping to be precisely what we need". diff --git a/_posts/2021-12-03-elixir-v1-13-0-released.markdown b/_posts/2021-12-03-elixir-v1-13-0-released.markdown new file mode 100644 index 000000000..486d27699 --- /dev/null +++ b/_posts/2021-12-03-elixir-v1-13-0-released.markdown @@ -0,0 +1,153 @@ +--- +layout: post +title: Elixir v1.13 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.13 is out with a focus on developer tooling +--- + +Elixir v1.13 has just been released! + +Generally speaking, new Elixir versions include improvements to its primary API, the one Elixir developers use every day, and also to the foundation that powers its tooling. In this release, however, it coincided that most new functionality centers around Elixir tooling. The result is a series of quality of life improvements that will impact Elixir developers immediately as well as in the long term. + +Let's check them out! + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, `noscript` tags with the proper links will be shown. + +## Semantic recompilation + +The feature that will most and immediately benefit all Elixir developers is the series of improvements we have made to how the compiler tracks file contents. + +Generally speaking, once a file changes, it may lead to other files in your codebase to be recompiled. In previous versions, however, Elixir made no effort to understand which parts of the file changed. This meant the smallest of changes to certain files, such as configuration files, could trigger a full project recompilation. + +This release comes with a series of improvements that better understand how your files change. In particular: + + * An Elixir file is no longer considered as changed if its size and its digest stay the same. This avoids recompiling many files when switching or rebasing branches. + + * Changing your `mix.exs` will no longer trigger a full project recompilation, unless you specifically change the configurations used by the Elixir compiler (`:elixirc_paths` and `:elixirc_options`). + + * Changing compile-time configuration files (`config/config.exs` and any other files imported from it) now only recompiles the project files that depend on the reconfigured applications, instead of a full project recompilation. However, if you change the configuration of your application itself, the whole project is still recompiled. + + * Adding, updating or removing a dependency now only recompiles the project files that depend on the modified dependency. + + * If your project has both Erlang and Elixir files, changing an Erlang file will now recompile only the Elixir files that depend on it. + +In a nutshell, Elixir went from triggering full recompilations whenever any of `mix.exs`, `config/config.exs`, `src/*`, and `mix.lock` changed on disk to semantic recompilations. Now it only fully recompiles when: + + * you change the compilation options in `mix.exs` + * you change the configuration for the current project in `config/config.exs` + +To give a more practical example, take a regular [Phoenix project](https://phoenixframework.org/). It is most likely divided in two main directories: `my_app` and `my_app_web`. Most of your usage of Phoenix' APIs will happen within the files in the `my_app_web` directory. However, if you bumped your Phoenix version or changed its configuration in previous Elixir versions, it would cause all files, in both directories, to be recompiled. With these changes, the recompilation should affect mostly the files in `my_app_web`. + +> To further clarify, the Elixir compiler is not tracking directories. It is just a consequence of how Phoenix projects are organized that most dependencies to Phoenix are within `my_app_web`. + +## Code fragments + +The [`Code`](https://hexdocs.pm/elixir/Code.html) module got a companion module called [`Code.Fragment`](https://hexdocs.pm/elixir/Code.Fragment.html). + +The `Code` module works with complete code. For example, its functions will consider the snippet `123 +` as invalid, since the right-hand side of `+` is missing. However, our tooling, such as editors, REPLs, and code notebooks must still parse and understand such snippets, in order to provide code completion, argument suggestion, etc. + +That's the goal of the `Code.Fragment` module. It contains different heuristics to analyze and return context informational of code fragments, which are code snippets that may be incomplete. + +To better show the benefits of said improvements, let's talk about IEx, Elixir's interactive shell. IEx has been rewritten to use `Code.Fragment` and, in the process, it gained new functionality as part of its autocompletion system (available by hitting TAB). For example, it can now autocomplete sigils, used to [create regexes](https://hexdocs.pm/elixir/Kernel.html#sigil_r/2) or [lists of words](https://hexdocs.pm/elixir/Kernel.html#sigil_w/2), and their terminators: + + + +Similarly, you can now autocomplete struct names and their fields: + + + +Overall, we hope the `Code.Fragment` module will become the shared foundation to power many of the tools in the ecosystem. We have also added new reflection APIs to [`Module`](https://hexdocs.pm/elixir/Module.html), which can then be used to power code intelligence features. + +## mix xref + +[`mix xref`](https://hexdocs.pm/mix/Mix.Tasks.Xref.html) is a tool that analyzes relationships between files. By analyzing the compile-time and runtime dependencies between them, it allows developers to understand what has to be recompiled whenever a file changes. + +Elixir v1.13 comes with many improvements to `mix xref`, such as: + + * `mix xref graph` now supports `--label` to be set to "compile-connected", which returns all compile-time dependencies that lead to additional transitive dependencies. + + * A new `mix xref trace FILE` subcommand receives a file and returns all dependencies in said file, including the line and what caused said dependency (a function/macro call, an alias, a struct, etc). + + * All `mix xref` subcommands support the `--fail-above` flag, which allows you to enforce your project has at most a certain number of compile-time cycles, transitive compile-time dependencies, etc. This can be useful on Continuous Integration (CI) servers. + + * `mix xref graph` now supports multiple `--sink` and `--source` to be given. + +If you haven't used `mix xref` before, it may be hard to visualize what those changes mean. If you want to learn more, you can [watch the relevant section of my ElixirConf 2021 keynote](https://youtu.be/ydjx2kKHzrM?t=772) that includes a short introduction to `mix xref`. + +Those improvements came from direct feedback from the community. A special shout out to Marc-André Lafortune for the contributions and testing. + +## Extended code formatting + +Thanks to its sigils, Elixir provides the ability of embedding snippets in other languages inside its source code. One could use it to embed XML: + + ~X""" + + + """ + +Or even [Zig](https://ziglang.org/), [via the Zigler project](https://github.com/ityonemo/zigler): + + ~Z""" + /// nif: example_fun/2 + fn example_fun(value1: f64, value2: f64) bool { + return value1 > value2; + } + """ + +However, while you can format Elixir source code with [`mix format`](https://hexdocs.pm/mix/Mix.Tasks.Format.html), you could not format the code inside snippets. + +Elixir v1.13 solves this by adding plugins to `mix format`. Plugins can teach the formatter how to format new files and how to format sigils, via the `Mix.Tasks.Format` behaviour. + +For example, imagine that your project uses Markdown in two distinct ways: via a custom `~M` sigil and via files with the `.md` and `.markdown` extensions. A custom plugin would look like this: + +```elixir +defmodule MixMarkdownFormatter do + @behaviour Mix.Tasks.Format + + def features(_opts) do + [sigils: [:M], extensions: [".md", ".markdown"]] + end + + def format(contents, opts) do + # logic that formats markdown + end +end +``` + +Now any application can use your formatter as follows: + +```elixir +# .formatter.exs +[ + # Define the desired plugins + plugins: [MixMarkdownFormatter], + # Remember to update the inputs list to include the new extensions + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}", "posts/*.{md,markdown}"] +] +``` + +We are looking forward to see how this new functionality will be used by community, especially projects like [Surface](https://github.com/surface-ui/surface) and [Phoenix LiveView](https://github.com/phoenixframework/phoenix_live_view), which provide a templating language on top of the HTML markup. + +## Other bits and bytes + +`SyntaxError` and `TokenMissingError` were improved to show a code snippet whenever possible: + + $ elixir -e "hello + * world" + ** (SyntaxError) nofile:1:9: syntax error before: '*' + | + 1 | hello + * world + | ^ + +The `Code` module has also been augmented with two functions: [`Code.string_to_quoted_with_comments/2`](https://hexdocs.pm/elixir/Code.html#string_to_quoted_with_comments/2) and [`Code.quoted_to_algebra/2`](https://hexdocs.pm/elixir/Code.html#quoted_to_algebra/2). Those functions allow someone to retrieve the Elixir AST with their original source code comments, and then convert this AST to formatted code. In other words, those functions provide a wrapper around the Elixir Code Formatter, supporting developers who wish to create tools that directly manipulate and custom format Elixir source code. + +`elixir --short-version` has been added to quickly get the Elixir version, without booting the Erlang VM. The `Task` module includes performance optimizations and [new](https://hexdocs.pm/elixir/Task.html#ignore/1) [functions](https://hexdocs.pm/elixir/Task.html#completed/1). Finally, `mix test --profile-require=time` has been added to debug loading times of test suites and the recently added [`Mix.install/2`](https://hexdocs.pm/mix/Mix.html#install#2) has been improved with new options and environment variables. + +## Learn more + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.13.0). You can also [watch my ElixirConf 2021 keynote about Elixir v1.13](https://youtu.be/ydjx2kKHzrM) to learn more. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2022-09-01-elixir-v1-14-0-released.markdown b/_posts/2022-09-01-elixir-v1-14-0-released.markdown new file mode 100644 index 000000000..7ae52d259 --- /dev/null +++ b/_posts/2022-09-01-elixir-v1-14-0-released.markdown @@ -0,0 +1,192 @@ +--- +layout: post +title: Elixir v1.14 released +authors: +- Andrea Leopardi +category: Releases +excerpt: Elixir v1.14 is out with a focus on debugging and developer experience +--- + +Elixir v1.14 has just been released. 🎉 + +Let's check out new features in this release. Like many of the past Elixir releases, this one has a strong focus on developer experience and developer happiness, through improvements to debugging, new debugging tools, and improvements to term inspection. Let's take a quick tour. + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, `noscript` tags with the proper links will be shown. + +## `dbg` + +[`Kernel.dbg/2`](https://hexdocs.pm/elixir/Kernel.html#dbg/2) is a new macro that's somewhat similar to [`IO.inspect/2`](https://hexdocs.pm/elixir/IO.html#inspect/2), but specifically tailored for **debugging**. + +When called, it prints the value of whatever you pass to it, plus the debugged code itself as well as its location. + + + +`dbg/2` can do more. It's a macro, so it *understands Elixir code*. You can see that when you pass a series of `|>` pipes to it. `dbg/2` will print the value for every step of the pipeline. + + + +## IEx + dbg + +Interactive Elixir (IEx) is Elixir's shell (also known as REPL). In v1.14, we have improved IEx breakpoints to also allow line-by-line stepping: + + + +We have also gone one step further and integrated this new functionality with `dbg/2`. + +`dbg/2` supports **configurable backends**. IEx automatically replaces the default backend by one that halts the code execution with IEx: + + + +We call this process "prying", as you get access to variables and imports, but without the ability to change how the code actually executes. + +This also works with pipelines: if you pass a series of `|>` pipe calls to `dbg` (or pipe into it at the end, like `|> dbg()`), you'll be able to step through every line in the pipeline. + + + +You can keep the default behavior by passing the `--no-pry` option to IEx. + +## dbg in Livebook + +[Livebook](https://livebook.dev/) brings the power of computation notebooks to Elixir. + +As an another example of the power behind `dbg`, the Livebook team has implemented a visual representation for `dbg` as a backend, where it prints each step of the pipeline as its distinct UI element. You can select an element to see its output or even re-order and disable sections of the pipeline on the fly: + + + +## PartitionSupervisor + +[`PartitionSupervisor`](https://hexdocs.pm/elixir/PartitionSupervisor.html) implements a new supervisor type. It is designed to help when you have a single supervised process that becomes a bottleneck. If that process' state can be easily partitioned, then you can use `PartitionSupervisor` to supervise multiple isolated copies of that process running concurrently, each assigned its own partition. + +For example, imagine you have a `ErrorReporter` process that you use to report errors to a monitoring service. + +```elixir +# Application supervisor: +children = [ + # ..., + ErrorReporter +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +As the concurrency of your application goes up, the `ErrorReporter` process might receive requests from many other processes and eventually become a bottleneck. In a case like this, it could help to spin up multiple copies of the `ErrorReporter` process under a `PartitionSupervisor`. + +```elixir +# Application supervisor +children = [ + {PartitionSupervisor, child_spec: ErrorReporter, name: Reporters} +] +``` + +The `PartitionSupervisor` will spin up a number of processes equal to `System.schedulers_online()` by default (most often one per core). Now, when routing requests to `ErrorReporter` processes we can use a `:via` tuple and route the requests through the partition supervisor. + +```elixir +partitioning_key = self() +ErrorReporter.report({:via, PartitionSupervisor, {Reporters, partitioning_key}}, error) +``` + +Using `self()` as the partitioning key here means that the same process will always report errors to the same `ErrorReporter` process, ensuring a form of back-pressure. You can use any term as the partitioning key. + +### A common example + +A common and practical example of a good use case for `PartitionSupervisor` is partitioning something like a `DynamicSupervisor`. When starting many processes under it, a dynamic supervisor can be a bottleneck, especially if said processes take a long time to initialize. Instead of starting a single `DynamicSupervisor`, you can start multiple: + +```elixir +children = [ + {PartitionSupervisor, child_spec: DynamicSupervisor, name: MyApp.DynamicSupervisors} +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +Now you start processes on the dynamic supervisor for the right partition. For instance, you can partition by PID, like in the previous example: + +```elixir +DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {MyApp.DynamicSupervisors, self()}}, + my_child_specification +) +``` + +## Improved errors on binaries and evaluation + +Erlang/OTP 25 improved errors on binary construction and evaluation. These improvements apply to Elixir as well. Before v1.14, errors when constructing binaries would often be hard-to-debug, generic "argument errors". Erlang/OTP 25 and Elixir v1.14 provide more detail for easier debugging. This work is part of [EEP 54](https://www.erlang.org/eeps/eep-0054). + +Before: + +```elixir +int = 1 +bin = "foo" +int <> bin +#=> ** (ArgumentError) argument error +``` + +Now: + +```elixir +int = 1 +bin = "foo" +int <> bin +#=> ** (ArgumentError) construction of binary failed: +#=> segment 1 of type 'binary': +#=> expected a binary but got: 1 +``` + +Code evaluation (in IEx and Livebook) has also been improved to provide better error reports and stacktraces. + +## Slicing with Steps + +Elixir v1.12 introduced **stepped ranges**, which are ranges where you can specify the "step": + +```elixir +Enum.to_list(1..10//3) +#=> [1, 4, 7, 10] +``` + +Stepped ranges are particularly useful for numerical operations involving vectors and matrices (see [Nx](https://github.com/elixir-nx/nx), for example). However, the Elixir standard library was not making use of stepped ranges in its APIs. Elixir v1.14 starts to take advantage of steps with support for stepped ranges in a couple of functions. One of them is [`Enum.slice/2`](https://hexdocs.pm/elixir/Enum.html#slice/2): + +```elixir +letters = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] +Enum.slice(letters, 0..5//2) +#=> ["a", "c", "e"] +``` + +[`binary_slice/2`](https://hexdocs.pm/elixir/Kernel.html#binary_slice/2) (and [`binary_slice/3`](https://hexdocs.pm/elixir/Kernel.html#binary_slice/3) for completeness) has been added to the `Kernel` module, that works with bytes and also support stepped ranges: + +```elixir +binary_slice("Elixir", 1..5//2) +#=> "lxr" +``` + +## Expression-based Inspection and `Inspect` Improvements + +In Elixir, it's conventional to implement the `Inspect` protocol for opaque structs so that they're inspected with a special notation, resembling this: + +```elixir +MapSet.new([:apple, :banana]) +#MapSet<[:apple, :banana]> +``` + +This is generally done when the struct content or part of it is private and the `%name{...}` representation would reveal fields that are not part of the public API. + +The downside of the `#name<...>` convention is that *the inspected output is not valid Elixir code*. For example, you cannot copy the inspected output and paste it into an IEx session. + +Elixir v1.14 changes the convention for some of the standard-library structs. The `Inspect` implementation for those structs now returns a string with a valid Elixir expression that recreates the struct when evaluated. In the `MapSet` example above, this is what we have now: + +```elixir +fruits = MapSet.new([:apple, :banana]) +MapSet.put(fruits, :pear) +#=> MapSet.new([:apple, :banana, :pear]) +``` + +The `MapSet.new/1` expression evaluates to exactly the struct that we're inspecting. This allows us to hide the internals of `MapSet`, while keeping it as valid Elixir code. This expression-based inspection has been implemented for `Version.Requirement`, `MapSet`, and `Date.Range`. + +Finally, we have improved the `Inspect` protocol for structs so that fields are inspected in the order they are declared in `defstruct`. The option `:optional` has also been added when deriving the `Inspect` protocol, giving developers more control over the struct representation. See [the updated documentation for `Inspect`](https://hexdocs.pm/elixir/Inspect.html) for a general rundown on the approaches and options available. + +## Learn more + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.14.0). + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Happy debugging! diff --git a/_posts/2022-10-05-my-future-with-elixir-set-theoretic-types.markdown b/_posts/2022-10-05-my-future-with-elixir-set-theoretic-types.markdown new file mode 100644 index 000000000..11f8d06c6 --- /dev/null +++ b/_posts/2022-10-05-my-future-with-elixir-set-theoretic-types.markdown @@ -0,0 +1,140 @@ +--- +layout: post +title: "My Future with Elixir: set-theoretic types" +authors: +- José Valim +category: Announcements +excerpt: We announce and explore the possibilities for bringing set-theoretic types into Elixir. +--- + +*This is article contains excerpts from my keynotes at [ElixirConf Europe 2022](https://www.youtube.com/watch?v=Jf5Hsa1KOc8) and [ElixirConf US 2022](https://www.youtube.com/watch?v=KmLw58qEtuM).* + +In May 2022, we have celebrated 10 years since Elixir v0.5, the first public release of Elixir, was announced. + +At such occasions, it may be tempting to try to predict how Elixir will look in 10 years from now. However, I believe that would be a futile effort, because, 10 years ago, I would never have guessed Elixir would have gone [beyond excelling at web development](https://phoenixframework.org/), but also into domains such as [embedded software](https://www.nerves-project.org/) and making inroads into machine learning and data analysis with projects such as [Nx (Numerical Elixir)](https://github.com/elixir-nx/nx), [Explorer](https://github.com/elixir-nx/explorer), [Axon](https://github.com/elixir-nx/axon), and [Livebook](https://livebook.dev/) ([here is a summary of the main Numerical Elixir projects](https://github.com/elixir-nx/). Elixir was designed to be extensible and how it will be extended has always been a community effort. + +For these reasons, I choose to focus on *My Future* with Elixir. Those are the projects I am personally excited about and working on alongside other community members. The topic of today's article is type systems, as discussed in my ElixirConf EU presentation in May 2022. + +## The elephant in the room: types + +Throughout the years, the Elixir Core Team has addressed the biggest needs of the community. [Elixir v1.6 introduced the Elixir code formatter](https://elixir-lang.org/blog/2018/01/17/elixir-v1-6-0-released/), as the growing community and large teams saw an increased need for style guides and conventions around large codebases. + +[Elixir v1.9 shipped with built-in support for releases](https://elixir-lang.org/blog/2019/06/24/elixir-v1-9-0-released/): self-contained archives that consist of your application code, all of its dependencies, plus the whole Erlang Virtual Machine (VM) and runtime. The goal was to address the perceived difficulty in deploying Elixir projects, by bringing tried approaches from both Elixir and Erlang communities into the official tooling. This paved the way to future automation, such as `mix phx.gen.release`, which automatically generates a Dockerfile tailored to your Phoenix applications. + +Given our relationship with the community, it would be disingenuous to talk about my future with Elixir without addressing what seems to be the biggest community need nowadays: static typing. However, when the community asks for static typing, what are we effectively expecting? And what is the Elixir community to gain from it? + +## Types and Elixir + +Different programming languages and platforms extract different values from types. These values may or may not apply to Elixir. + +For example, different languages can extract performance benefits from types. However, Elixir still runs on the Erlang VM, which is dynamically typed, so we should not expect any meaningful performance gain from typing Elixir code. + +Another benefit of types is to _aid_ documentation (emphasis on the word _aid_ as I don't believe types replace textual documentation). Elixir already reaps similar benefits from [typespecs](https://hexdocs.pm/elixir/typespecs.html) and I would expect an integrated type system to be even more valuable in this area. + +However, the upsides and downsides of static typing become fuzzier and prone to exaggerations once we discuss them in the context of code maintenance, in particular when comparing types with other software verification techniques, such as tests. In those situations, it is common to hear unrealistic claims such as "a static type system would catch 80% of my Elixir bugs" or that "you need to write fewer tests once you have static types". + +While [I explore why I don't believe those claims are true during the keynote](https://www.youtube.com/watch?v=Jf5Hsa1KOc8), saying a static type system helps catch bugs is not helpful unless we discuss exactly the type of bugs it is supposed to identify, and that's what we should focus on. + +For example, Rust's type system helps prevent bugs such as deallocating memory twice, dangling pointers, data races in threads, and more. But adding such type system to Elixir would be unproductive because those are not bugs that we run into in the first place, as those properties are guaranteed by the garbage collector and the Erlang runtime. + +This brings another discussion point: a type system naturally restricts the amount of code we can write because, in order to prove certain properties about our code, certain styles have to be rejected. However, I would prefer to avoid restricting the expressive power of Elixir, because I am honestly quite happy with the language semantics (which we mostly inherited from Erlang). + +For Elixir, the benefit of a type system would revolve mostly around contracts. If function `caller(arg)` calls a function named `callee(arg)`, we want to guarantee that, as both these functions change over time, that `caller` is passing valid arguments into `callee` and that the `caller` properly handles the return types from `callee`. + +This may seem like a simple guarantee to provide, but we'd run into tricky scenarios even on small code samples. For example, imagine that we define a `negate` function, that negates numbers. One may implement it like this: + +```elixir +def negate(x) when is_integer(x), do: -x +``` + +We could then say `negate` has the type `integer() -> integer()`. + +With our custom negation in hand, we can implement a custom subtraction: + +```elixir +def subtract(a, b) when is_integer(a) and is_integer(b) do + a + negate(b) +end +``` + +This would all work and typecheck as expected, as we are only working with integers. However, imagine in the future someone decides to make `negate` polymorphic, so it also negates booleans: + +```elixir +def negate(x) when is_integer(x), do: -x +def negate(x) when is_boolean(x), do: not x +``` + +If we were to naively say that `negate` now has the type `integer() | boolean() -> integer() | boolean()`, we would now get a false positive warning in our implementation of subtract: + +```elixir +Type warning: + + | + | def subtract(a, b) when is_integer(a) and is_integer(b) do + | a + negate(b) + ^ the operator + expects integer(), integer() as arguments, + but the second argument can be integer() | boolean() +``` + +**So we want a type system that can type contracts between functions but, at the same time, avoids false positives and does not restrict the Elixir language**. Balancing those trade-offs is not only a technical challenge but also one that needs to consider the needs of the community. The [Dialyzer project](https://www.erlang.org/doc/man/dialyzer.html), implemented in Erlang and available for Elixir projects, chose to have no false positives. However, that implies certain bugs may not be caught. + +At this point in time, it seems the overall community would prefer a system that flags more potential bugs, even if it means more false positives. This may be particularly tricky in the context of Elixir and Erlang because I like to describe them as [_assertive languages_](https://dashbit.co/blog/writing-assertive-code-with-elixir): we write code that will crash in face of unexpected scenarios because we rely on supervisors to restart parts of our application whenever that happens. This is the foundation of building self-healing and fault-tolerant systems in those languages. + +On the other hand, this is what makes a type system for Erlang/Elixir so exciting and unique: the ability to deal with failure modes both at compile-time and runtime elegantly. Because at the end of the day, regardless of the type system of your choice, you will run into unexpected scenarios, especially when interacting with external resources such as the filesystem, APIs, distributed nodes, etc. + +## The big announcement + +This brings me to the big announcement from ElixirConf EU 2022: **we have an on-going PhD scholarship to research and develop a type system for Elixir based on set-theoretic types**. Guillaume Duboc (PhD student) is the recipient of the scholarship, lead by Giuseppe Castagna (Senior Resercher) with support from José Valim (that's me). + +The scholarship is a partnership between the [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). It is sponsored by Supabase ([they are hiring!](https://supabase.com/company)), Fresha ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), and [Dashbit](https://dashbit.co/), all heavily invested in Elixir's future. + +## Why set-theoretic types? + +We want a type system that can elegantly model all of Elixir idioms and, at a first glance, set-theoretic types were an excellent match. In set-theoretic types, we use set operations to define types and ensure that the types satisfy the associativity and distributivity properties of the corresponding set-theoretic operations. + +For example, numbers in Elixir can be integers _or_ floats, therefore we can write them as the union `integer() | float()` (which is equivalent to `float() | integer()`). + +Remember the `negate` function we wrote above? + +```elixir +def negate(x) when is_integer(x), do: -x +def negate(x) when is_boolean(x), do: not x +``` + +We could think of it as a function that has both types `(integer() -> integer())` and `(boolean() -> boolean())`, which is as an intersection. This would naturally solve the problem described in the previous section: when called with an integer, it can only return an integer. + +We also have a data-structure called atoms in Elixir. They uniquely represent a value which is given by their own name. Such as `:sunday` or `:banana`. You can think of the type `atom()` as the set of all atoms. In addition, we can think of the values `:sunday` and `:banana` as subtypes of `atom()`, as they are contained in the set of all atoms. `:sunday` and `:banana` are also known as singleton types (as they are made up of only one value). + +In fact, we could even consider each integer to be a singleton type that belongs to the `integer()` set. The choice of which values will become singletons in our type system will strongly depend on the trade-offs we defined in the previous sections. The type system also has to be gradual, as typed Elixir code may interact with untyped Elixir code and vice-versa. + +Personally, I find set-theoretical types an elegant and accessible approach to reason about types. At the end of the day, an Elixir developer won't have to think about intersections when writing a function with multiple clauses, but the modelling is straight-forward if they are ever to look under the hood. + +Despite the initial fit between Elixir semantics and set-theoretic types, there are open questions and existing challenges in putting the two together. Here are some examples: + + * Elixir has [an expressive collection of idioms used in pattern matching and guards](https://hexdocs.pm/elixir/patterns-and-guards.html), can we map them all to set-theoretic types? + + * Elixir associative data structures, [called maps](https://hexdocs.pm/elixir/Map.html), can be used both as records and as dictionaries. Would it be possible to also type them with a unified foundation? + + * Gradual type systems must introduce runtime type checks in order to remain sound. However, those type checks will happen in addition to the checks already done by the Erlang VM, which can degrade performance. Therefore, is it possible to leverage the existing runtime checks done by the Erlang VM so the resulting type system is still sound? + +Those challenges are precisely what makes me excited to work with Giuseppe Castagna and Guillaume Duboc, as we believe it is important to formalize those problems and their solutions, before we dig deep into the implementation. To get started with set-theoretic types, I recommend [Programming with union, intersection, and negation types by Giuseppe Castagna](https://www.irif.fr/~gc/papers/set-theoretic-types-2022.pdf). + +Finally, it is important to note there are areas we don't plan to tackle at the moment, such as typing of messages between processes. + +## Expectations and roadmap + +At this point, you may be expecting that Elixir will certainly become a gradually typed language at some moment in its future. However, it is important to note this may not be the case, as there is a long road ahead of us. + +One of the challenges in implementing a type system - at least for someone who doesn't have the relevant academic background like myself - is that it feels like a single indivisible step: you take a language without a type system and at the end you have one, without much insight or opportunity for feedback in the middle. Therefore we have been planning to incorporate the type system into Elixir in steps, which I have been referring to as "a gradual gradual type system": one where we add gradual types to the language gradually. + +The first step, the one we are currently working on, is to leverage the existing type information found in Elixir programs. As previously mentioned, [we write assertive code](https://dashbit.co/blog/writing-assertive-code-with-elixir) in Elixir, which means there is a lot of type information in patterns and guards. We want to lift this information and use it to type check existing codebases. The Erlang compiler already does so to improve performance within a single module and we want to eventually do so across modules and applications too. + +During this phase, Elixir developers won't have to change a single line of code to leverage the benefits of the type system. Of course, we will catch only part of existing bugs, but this will allows us to stress test, benchmark, and collect feedback from developers, making improvements behind the scenes (or even revert the whole thing if we believe it won't take us where we expect). + +The next step is to introduce typed structs into the language, allowing struct types to propagate throughout the system, as you pattern match on structs throughout the codebase. In this stage we will introduce a new API for defining structs, yet to be discussed, and developers will have to use the new API to reap its benefits. + +Finally, once we are happy with the improvements and the feedback collected, we can migrate to introduce a new syntax for typing function signatures in Elixir codebases, including support for more advanced features such as polymorphic types. Those will allow us to type complex constructs such as the ones found in the `Enum` module. + +The important point to keep in mind is that those features will be explored and developed in steps, with plenty of opportunity to gather community feedback. I also hope our experience may be useful to other ecosystems who wish to gradually introduce type systems into existing programming languages, in a way that feels granular and participative. + +Thank you for reading and see you in future updates. diff --git a/_posts/2022-12-22-cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience.markdown b/_posts/2022-12-22-cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience.markdown new file mode 100644 index 000000000..505bf15bf --- /dev/null +++ b/_posts/2022-12-22-cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience.markdown @@ -0,0 +1,188 @@ +--- +layout: post +title: "Cheatsheets and other 8 ExDoc features that improve the developer experience" +authors: +- Hugo Baraúna +category: Announcements +excerpt: This post explains the motivation behind the new ExDoc Cheatsheet feature. It also highlights other ExDoc features that show how ExDoc has been evolving to make the documentation experience in Elixir better and better. +--- + +ExDoc has a cool new feature, [cheatsheets](https://hexdocs.pm/ex_doc/cheatsheet.html)! + +In this blog post, we'll explain what that new feature is and the motivation behind it. We'll also take the opportunity to highlight other ExDoc features that show how it has been evolving to make the documentation experience in Elixir better and better. + +## What are ExDoc cheatsheets and how they improve the documentation experience + +ExDoc's cheatsheets are Markdown files with the `.cheatmd` extension. You can see [an example](https://hexdocs.pm/ecto/crud.html) of how the Ecto project is using them. + +Writing and reading cheatsheets is not exactly new to developers. What ExDoc brings to the table is the possibility of integrating cheatsheets alongside the rest of the documentation of an Elixir project, instead of hosting them in a different place. + +Developers need different kinds of documentation at different times. When one is learning about a new library, a guide format is proper. When one needs to know if a library can solve a specific problem, an API reference can be more appropriate. When someone wants to remember a couple of functions they already used from that library, a cheatsheet could be more practical. + +Imagine if you had to go to a different place for every type of documentation you're looking for. That would make a very fragmented experience, not only for readers of documentation but also for writers. + +ExDoc cheatsheets represent one step further in the direction of making documentation in Elixir an even more comprehensive and integrated experience. + +ExDoc cheatsheets are inspired by [devhints.io](https://devhints.io) from [Rico Sta. Cruz](https://twitter.com/rstacruz), and were contributed by [Paulo Valim](https://twitter.com/paulovalim) and [Yordis Prieto](https://twitter.com/alchemist_ubi). + +## Eight features that show how ExDoc has improved developer experience over time + +We added cheatsheets to ExDoc because we value developer experience and believe documentation is a core aspect of it. + +Since the beginning, one of Elixir's principles is that documentation should be a first-class citizen. What this idea means to us is that documentation should be easy to write and easy to read. ExDoc has been continuously evolving over the years, guided by this principle. + +Here are some of the features added to ExDoc over the years that make reading and writing documentation in Elixir a joy. + +### Beautiful and usable design + +As developers, we may not have the skill to make beautifully designed UIs. That doesn't mean we don't appreciate it. Here's what documentation generated with ExDoc looked like almost ten years ago, with its original layout based on [YARD](https://yardoc.org/): + +![Screenshot of the Phoenix v0.5.0 documentation generated with an early version +of ExDoc](https://i.imgur.com/O9xKjR8.jpg) + +Here's what it looks like today: + +![Screenshot of the Phoenix v1.6.15 documentation generated with current +ExDoc](https://i.imgur.com/ZKI1T23.png) + +The evolution of ExDoc's design helped documentation be more visually appealing and easier to read and navigate. + +### Links to source code + +Sometimes you're reading the documentation of a library, and you want to know more about the implementation of a function. Or, you found something in the documentation that could be improved and wants to help. In those situations, it's helpful to go from the documentation to the source code. ExDoc makes that dead easy. For every module, function, or page, ExDoc gives you a link that you can click to go directly to the project's source code on GitHub: + +![Short screencast of a user clicking on the "link to source code" button on the +documentation for a function](https://i.imgur.com/PXvoeDk.gif) + +### Guides + +One of the most common formats of library documentation is an API reference. But depending on your needs, that's not the most approachable format. For example, it's not optimal when you're just getting started with a library or when you want to learn how to solve a specific problem using it. That's why ExDoc allows writing other types of docs besides API references, like *"Getting started" guides* or *How-tos*. + +Look at how [Ecto's documentation](https://hexdocs.pm/ecto/getting-started.html) uses that, for example: + +![Screencast of a user exploring the guides in the Ecto +documentation](https://i.imgur.com/KInZb4x.gif) + +### Custom grouping of modules, functions, and pages in the sidebar + +Sometimes your library has dozens of modules. Sometimes, one given module has a large API surface area. In those situations, showing the list of functions as a single large list may not be the most digestible way to be consumed. For those cases, ExDoc allows modules, functions, or extra pages to be grouped in the sidebar in a way that makes more sense semantically. + +Here's an example of how Ecto use grouped functions for its `Repo` module: + +![Screenshot of the sidebar of the Ecto documentation, showing grouped functions +in the `Ecto.Repo` module](https://i.imgur.com/ZE7N312.png) + +Instead of listing the ~40 functions of `Ecto.Repo` as a single extensive list, it presents them grouped by five cohesive topics: + +- Query API +- Schema API +- Transaction API +- Runtime API +- User callbacks + +The same functionality is available for modules and pages (guides, how-tos, and so on). Phoenix is a [good example](https://hexdocs.pm/phoenix/overview.html) of how that's used. + +### Full-text search + +Sometimes you don't know or don't remember the name of the function that you're looking for. For example, let's say you're looking for a function for dealing with file system directories. + +Although there's no function or module called "directory" in Elixir, when you type "directory" in [Elixir's documentation](https://hexdocs.pm/elixir/search.html?q=directory), it will return all the entries that have the word "directory" inside the documentation. It will even return entries with variations of the word "directory", like "directories", doing a fuzzy search. + +![Screenshot of the result of searching for "directory" in the Elixir +documentation](https://i.imgur.com/IHHuej8.png) + +The search bar also supports autocompletion for module and function names: + +![Screencast of a user typing the word "Enum" in the search bar of Elixir's +documentation and letting it autocomplete the module. Then, the user types +"Range" and both modules and functions show +up.](https://i.imgur.com/2cmsuDi.gif) + +The best part is that full-text search is fully implemented on the client-side, which means ExDoc pages can be fully hosted as static websites (for example on GitHub Pages). + +### Keyboard shortcuts to navigate to docs of other Hex packages + +It's common for an application to have dependencies. While coding, we usually need to read the documentation of more than one of those dependencies. + +One solution is to keep a window open for the documentation of each dependency. However, ExDoc offers another option: a keyboard shortcut to search and go to another package documentation within the same window. + +Here's what it looks like: + +![Screencast of a user enabling the `g` shortcut to search through dependencies +documentation and then using it to search for "phoenix_live" in the +documentation for Nerves.](https://i.imgur.com/I9uJxUF.gif) + +There are more keyboard shortcuts to help you navigate within and between documentation: + +![Screenshot of the keyboard shortcuts that you can enable in +ExDoc](https://i.imgur.com/qdoNUx9.png) + +### A version dropdown to switch to other versions + +Keeping our application updated with the latest versions of all its dependencies can be challenging. So, it's common to need to look at the documentation of an older version of a library we're using. ExDoc makes it simple to do that. + +When you access the documentation of a project, there's a dropdown that you can use to select the version you're looking for: + +![Screencast of a user typing the version dropdown under the application name in +the "timex" documentation, revealing all the +versions.](https://i.imgur.com/1krcY5g.gif) + +### Livebook integration + +[Livebook](https://livebook.dev/) is a web application for writing interactive and collaborative code notebooks in Elixir. + +One of the ways Elixir developers have been using Livebook is for documentation. Because of its interactivity capabilities, it enables the reader to play with the code right inside the documentation, which makes it great for tutorials and augmenting the user experience. + +With that in mind, ExDoc offers the possibility of integrating Livebook notebooks. That means one can host Livebook-based documentation together with the API reference. + +Here's an [example of using Livebook inside ExDoc for writing a Usage Guide](https://hexdocs.pm/req_sandbox/usage.html): + +![Screencast of a user navigating through the "req_sandbox" documentation, +finding a Livebook, clicking "Run in Livebook", and using the Livebook that +opens up on their local machine.](https://i.imgur.com/FxOLs0Y.gif) + +### Bonus: Erlang support + +[EEP 48](https://www.erlang.org/eeps/eep-0048) proposed a standardized way for how BEAM languages could store API documentation. This allows any BEAM language to read documentation generated by each other. + +By leveraging that work, ExDoc can generate documentation for an Erlang project. For example, Telemetry is a library written in Erlang that has [its documentation](https://hexdocs.pm/telemetry/readme.html) generated with ExDoc. + +![Screenshot of "telemetry" documentation generated with +ExDoc](https://i.imgur.com/C4Idbuh.png) + +By using ExDoc to also generate documentation for Erlang-based projects, we can have more consistency in the user experience along the BEAM ecosystem. See the great [`rebar3_ex_doc`](https://hexdocs.pm/rebar3_ex_doc/) plugin to get started. + +### Bonus: Doctests + +When writing documentation, it's helpful to offer code examples. For instance, here's the documentation of the `Enum.any?/1` function from Elixir's standard library: + +```elixir +@doc """ +Returns `true` if at least one element in `enumerable` is truthy. + +When an element has a truthy value (neither `false` nor `nil`) iteration stops +immediately and `true` is returned. In all other cases `false` is returned. + +## Examples + + iex> Enum.any?([false, false, false]) + false + + iex> Enum.any?([false, true, false]) + true + + iex> Enum.any?([]) + false + +""" +``` + +To ensure examples do not get out of date, Elixir's test framework ExUnit provides a feature called **doctests**. This allows developers to test the examples in their documentation. Doctests work by parsing out code samples starting with `iex>` from the documentation. + +Although this is not a feature of ExDoc, it is an essential part of Elixir's developer and documentation experience. + +## Wrap up + +As we saw, ExDoc has evolved a lot throughout the years! As it continues to evolve into a more and more comprehensive documentation tool, we want to enable developers to keep investing more time writing the documentation itself instead of needing to spend time building custom documentation tools and websites. The best part is that all you need to do to leverage many of those features is to simply document your code using the `@doc` attribute! + +Here's to a continuously improving documentation experience for the next years. diff --git a/_posts/2023-03-09-embedded-and-cloud-elixir-at-sparkmeter.markdown b/_posts/2023-03-09-embedded-and-cloud-elixir-at-sparkmeter.markdown new file mode 100644 index 000000000..2a5fa73ce --- /dev/null +++ b/_posts/2023-03-09-embedded-and-cloud-elixir-at-sparkmeter.markdown @@ -0,0 +1,125 @@ +--- +layout: post +title: "Embedded and cloud Elixir for grid-management at Sparkmeter" +authors: +- Hugo Baraúna +category: Elixir in Production +excerpt: A case study of how Elixir is being used at SparkMeter. +logo: /images/cases/logos/sparkmeter.png +tags: energy iot nerves +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all](https://elixir-lang.org/cases.html) cases we have published so far.* + +[SparkMeter](https://www.sparkmeter.io/) is a company on a mission to increase access to electricity. They offer grid-management solutions that enable utilities in emerging markets to run financially-sustainable efficient, and reliable systems. + +Elixir has played an important role in simplifying SparkMeter systems by providing a unified developer experience across their products. Elixir's versatility in different domains, such as embedded software, data processing, and HTTP APIs, proved to be a valuable asset to a team who aims to release robust products quickly and confidently. + +Two of their products are smart electrical meters and grid-management software. These can be used to measure electricity usage, gather health information about an electrical grid, and manage billing. + +Here's an overview of their architecture: + +![SparkMeter architecture generation one](/images/cases/bg/sparkmeter-old-architecture.png) + +The meters are embedded devices responsible for collecting measures such as electricity usage. They communicate with each other via a mesh network and also communicate with the grid edge management unit. The grid edge management unit is an embedded system that receives and processes data from up to thousands of meters. The grid edge management unit also communicates with servers running in the cloud. Those servers send and receive data to the grid edge management units and process it for use by internal systems and user-facing software. + +## The challenge + +The infrastructure in which their embedded devices are deployed is not reliable. The cellular network used for communication between the ground and the cloud could fail, and the electricity supply to the embedded systems could go down. Therefore, their system needed to be fault-tolerant, and they needed to build equipment that didn't require constant field maintenance. + +In light of these requirements, they identified areas for improvement in the first generation of their product. One of the things they needed to improve was the development of a new grid edge management unit. Additionally, their product was mission-critical, so they wanted a technology they could confidently put into production and one that would not take more than a year of development and QA before releasing a new generation of their product. + +That's when they discovered Elixir and Nerves. + +## The trade-offs of adopting Elixir and Nerves +Nerves is an open-source platform that combines the Erlang virtual machine and Elixir ecosystem to build and deploy embedded systems. + +When considering the adoption of Elixir and [Nerves](https://nerves-project.org/), SparkMeter recognized many advantages the technologies offered. + +Elixir helped them meet the requirement of building a distributed and fault-tolerant system. That's because Elixir leverages the power of the Erlang VM and the OTP framework, which were designed with that requirement in mind. + +Regarding Nerves, they saw it as an entire ecosystem for doing embedded development with many advantages. For example, it has a good story for doing local development and going from that to [deploying](https://www.nerves-hub.org/) on an embedded device. It makes it easy to connect to an embedded device for iterative development. And it also enables fine-grained control of system boot, so they can handle scenarios when certain parts of the system won't start. + +That said, they had two concerns, the growth of Nerves and finding talent with expertise in the Elixir/Nerves stack. + +They wanted to ensure that Nerves would continue to grow. But they realized that even if it didn't, the benefits Nerves was already offering could give them a lot of leverage. Here are's what their senior VP of engineering, Jon Thacker, had to say about that: + +> Without Nerves, we would be on our own to figure out a lot. How to do distribution, the development environment, and how to support different architectures. So it really is a batteries-included framework for doing production-grade embedded systems. +> +> \- *Jon Thacker, Senior VP of Engineering* + +When we interviewed Jon for this case study, they had already been using Elixir and Nerves for more than two years. And with the benefit of hindsight, here's what he said about adopting Nerves: + +> Making sure that Nerves continued to grow was a concern. But it has done so and is showing a very positive trajectory. It was a calculated risk and, as it turns out, it was the correct choice. +> +> \- *Jon Thacker, Senior VP of Engineering* + +When it came to finding talent, they approached the problem in two ways. First, they started to build the pilot with a contractor to ensure that the staffing risk didn't affect their timeline. But they also wanted to have an internal team to take ownership of the product in the long term. So, shortly after finishing the first version of the new system, they hired two engineers with experience in Elixir, Michael Waud and Benjamin Milde. + +Besides hiring people with previous experience in Elixir, Jon noticed that training their embedded engineers in Elixir was also a viable option. Here's what he told us about that: + +> I'm traditionally an embedded engineer, and I only learned Elixir as part of this project. However, transferring my mental model was so easy that I do believe that we would be capable of training other embedded engineers as well. +> +> \- *Jon Thacker, Senior VP of Engineering* + +## The new system + +SparkMeter used Elixir for the ground (embedded) and cloud aspects of the new system they built. Here is an overview of the architecture: + +![SparkMeter architecture generation two](/images/cases/bg/sparkmeter-new-architecture.png) + +For the firmware of the grid edge management unit, they used Nerves. For the hardware, they built on top of a BeagleBone Black device. + +The communication between the grid edge management unit and the meters was via radio, using Rust to manage the radio hardware module inside the grid edge management unit. They used [Elixir Ports](https://hexdocs.pm/elixir/1.13.4/Port.html) to communicate with Rust and process the data from the meters. + +Elixir was also used for communication with the cloud servers via 3G or Edge. This communication required bandwidth usage optimization due to the cost of sending large volumes of data through the cellular network. They evaluated various solutions like REST, CoAP, MQTT, Kafka, and Websockets. Still, none fit their specific needs, so they created a custom protocol tailored to their use case, which involved designing a binary protocol and implementing a TCP server. Mike Waud discussed this in more detail in his talks at [ElixirConf 2021](https://www.youtube.com/watch?v=DJRL86mO4ks) and [2022](https://www.youtube.com/watch?v=BxTIUvyZHKw). + +The grid edge management unit also required a local web user interface that could be accessed on-site via Wi-Fi. For this, they used Phoenix and Liveview. + +The cloud aspect of the system is responsible for receiving data from the grid edge management units and sending control commands. It also runs a TCP server with their custom protocol, implemented in Elixir. The data received from the grid edge management units is stored in PostgreSQL and then consumed by a [Broadway-based](https://elixir-broadway.org/) data pipeline. + +The cloud system also exposes an HTTP API implemented with Phoenix. This API is consumed by other internal systems to interact with their PostgreSQL database. + +## Reaping the benefits + +During and after the development of the new generation of their system, SparkMeter observed many benefits. + +One of them was the reduction of the complexity of the grid edge management unit. The old version had more moving parts, using Ubuntu and Docker for the system level, Python/Celery and RabbitMQ for asynchronous processing, and Systemd for managing starting job processes. + +In the new version, they replaced all of that mainly with Elixir and Nerves. And for the parts where they needed tools that were not part of the BEAM stack, they could manage them like any other BEAM process by using [Elixir Ports](https://hexdocs.pm/elixir/1.13.4/Port.html). Here's what they said about that experience: + +> The new grid edge management unit has a very unified architecture. We can treat everything as an (Elixir) process. We have full control over the start and stop within a single ecosystem. It's just a very coherent storyline. +> +> \- *Jon Thacker, Senior VP Of Engineering* + +Another aspect they liked about Nerves was that it included security best practices. For example, they used SSL certificates on the client and the server side for communication between the ground and the cloud. Nerves made this easy through the [NervesKey component](https://github.com/nerves-hub/nerves_key), which enables the use of a hardware security module to protect the private key. Nerves also made it easy to keep up with system security patches, as the firmware generated by Nerves is a single bundle containing a minimal Linux platform and their application packaged as a [release](https://hexdocs.pm/mix/Mix.Tasks.Release.html). Here's what they said about security in Nerves: + +> It's easy enough to keep tracking upstream changes, so we're not getting behind the latest security patches. Nerves made that easy. Nerves just pushed us towards a good security model. +> +> \- *Jon Thacker, Senior VP Of Engineering* + +The communication between the ground and the cloud involved implementing a custom TCP server running in both parts of the system. Network programming is not an everyday task for many application developers, but Elixir helped them a lot with that: + +> I had never written a TCP client or a server before, it's just not something you even think about. But doing it in Elixir, particularly on the protocol level of sending binaries, was a pleasure to work with! Something that would be super tedious in an imperative language, with Elixir and pattern matching, is so clear! +> +> \- *Michael Waud, Senior Software Engineer* + +Another benefit they received from using Elixir on the ground and in the cloud was code reuse. For example, the encoding and decoding of their custom protocol were reused for both the embedded and cloud parts. + +> It would've been a much larger challenge if we hadn't been running Elixir in the cloud and on the grid edge management unit because we could write it once. The encoding and decoding we wrote once, we gained a lot from being able to share code. +> +> \- *Michael Waud, Senior Software Engineer* + +Michael also pointed out that by controlling the complete connection from the grid edge management unit up to the cloud, they could reduce bandwidth usage and improve resiliency, which were essential requirements for them. + +Finally, the new generation of their system also enabled them to release more often. Before, they were releasing new versions every quarter, but with the new system, they could release weekly when needed. + +## Summing up + +In conclusion, SparkMeter's adoption of Elixir and Nerves has led to many benefits for their mission-critical grid-management system. + +Elixir was used to design elegant solutions across data processing, HTTP APIs, and within the embedded space. This unified development model led to a more productive and robust environment, with less complexity and fewer moving parts. + +Additionally, the ability to control the entire connection from the ground to the cloud resulted in reduced bandwidth usage and improved resiliency. This fulfills essential requirements, given the diversity of conditions and locations the grid edge management unit may be deployed at. + +The new system also allowed for more frequent releases, enabling SparkMeter to respond quickly to their business needs. \ No newline at end of file diff --git a/_posts/2023-06-19-elixir-v1-15-0-released.markdown b/_posts/2023-06-19-elixir-v1-15-0-released.markdown new file mode 100644 index 000000000..ffc23b810 --- /dev/null +++ b/_posts/2023-06-19-elixir-v1-15-0-released.markdown @@ -0,0 +1,170 @@ +--- +layout: post +title: Elixir v1.15 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.15 with improvements to compilation and boot times. +--- + +Elixir v1.15 has just been released. 🎉 + +Elixir v1.15 is a smaller release with focused improvements +on compilation and boot times. This release also completes +our integration process with Erlang/OTP logger, bringing new +features such as log rotation and compression out of the box. + +You will also find additional convenience functions in `Code`, +`Map`, `Keyword`, all Calendar modules, and others. + +Finally, we are glad to welcome [Jean Klingler](https://github.com/sabiwara/) +as a member of the Elixir Core team. Thank you for your contributions! + +## Compile and boot-time improvements + +The last several releases brought improvements to compilation +time and this version is no different. In particular, Elixir +now caches and prunes load paths before compilation, ensuring your +project (and dependencies!) compile faster and in an environment +closer to production. + +In a nutshell, the Erlang VM loads modules from code paths. Each +application that ships with Erlang and Elixir plus each dependency +become an entry in your code path. The larger the code path, the +more work Erlang has to do in order to find a module. + +In previous versions, Mix would only add entries to the load paths. +Therefore, if you compiled 20 dependencies and you went to compile +the 21st, the code path would have 21 entries (plus all Erlang and +Elixir apps). This allowed modules from unrelated dependencies to +be seen and made compilation slower the more dependencies you had. +With this release, we will now prune the code paths to only the ones +listed as dependencies, bringing the behaviour closer to `mix release`. + +Furthermore, Erlang/OTP 26 allows us to start applications +concurrently and cache the code path lookups, decreasing the cost of +booting applications. The combination of Elixir v1.15 and Erlang/OTP 26 +should also reduce the boot time of applications, such as when starting +`iex -S mix` or running a single test with `mix test`. + +As an example, I have benchmarked [the Livebook application](https://github.com/livebook-dev/livebook) +on a M1 Max MacStudio across different Elixir and Erlang/OTP versions. +At the time of benchmarking, Livebook had ~200 source `.ex` files and +~35 dependencies. Compilation-times were improved by 16%: + +![Livebook compilation times](/images/contents/livebook-compile-1.15.png) + +Livebook saw an improvement of 30% on boot times: + +![Livebook boot times](/images/contents/livebook-boot-1.15.png) + +Different application will see different results. Our expectations +are the gains will be more meaningful the more dependencies you have, +the more files you have, and the more cores you have. We have even +received reports of up to 40% faster compilation times, although it +is yet unclear how generalizable this will be in practice. Note this +work does not improve the time to compile slow individual files. + +The compiler is also smarter in several ways: `@behaviour` declarations +no longer add compile-time dependencies and aliases in patterns and +guards add no dependency whatsoever, as no dispatching happens. Furthermore, +Mix now tracks the digests of `@external_resource` files, reducing the +amount of recompilation when swapping branches. Finally, dependencies +are automatically recompiled when their compile-time configuration changes, +providing a smoother development experience. + +##### Potential incompatibilities + +Due to the code path pruning, if you have an application or dependency +that does not specify its dependencies on Erlang/OTP and core Elixir applications, +which has always been erroneus behaviour, it may no longer compile +successfully in Elixir v1.15. You can temporarily disable code path pruning +by setting `prune_code_paths: false` in your `mix.exs`, although doing so +may lead to runtime bugs that are only manifested inside a `mix release`. + +## Compiler warnings and errors + +The Elixir compiler can now emit many errors for a single file, making +sure more feedback is reported to developers before compilation is aborted. + +In Elixir v1.14, an undefined function would be reported as: + + ** (CompileError) undefined function foo/0 (there is no such import) + my_file.exs:1 + +In Elixir v1.15, the new reports will look like: + + error: undefined function foo/0 (there is no such import) + my_file.exs:1 + + ** (CompileError) my_file.exs: cannot compile file (errors have been logged) + +A new function, called `Code.with_diagnostics/2`, has been added so this +information can be leveraged by editors, allowing them to point to several +errors at once. We have currently ongoing work and contribution to further +improve the compiler diagnostics in future Elixir releases. + +##### Potential incompatibilities + +As part of this effort, the behaviour where undefined variables were transformed +into nullary function calls, often leading to confusing error reports, has +been disabled during project compilation. You can invoke `Code.compiler_options(on_undefined_variable: :warn)` +at the top of your `mix.exs` to bring the old behaviour back. + +## Integration with Erlang/OTP logger + +This release provides additional features such as global logger +metadata and [file logging](https://hexdocs.pm/logger/Logger.html#module-erlang-otp-handlers) (with rotation and compression) out of the box! + +This release also soft-deprecates Elixir's Logger Backends in +favor of Erlang's Logger handlers. Elixir will automatically +convert your `:console` backend configuration into the new +configuration. Previously, you would set: + +```elixir +config :logger, :console, + level: :error, + format: "$time $message $metadata" +``` + +Which is now translated to the equivalent: + +```elixir +config :logger, :default_handler, + level: :error + +config :logger, :default_formatter, + format: "$time $message $metadata" +``` + +To replace the default console handler by one that writes to disk, +with log rotation and compression: + +```elixir +config :logger, :default_handler, + config: [ + file: ~c"system.log", + filesync_repeat_interval: 5000, + file_check: 5000, + max_no_bytes: 10_000_000, + max_no_files: 5, + compress_on_rotate: true + ] +``` + +Finally, the previous Logger Backends API is now soft-deprecated. +If you implement your own backends, you want to consider migrating to +[`:logger_backends`](https://github.com/elixir-lang/logger_backends) +in the long term. See the new [`Logger`](https://hexdocs.pm/logger) +documentation for more information on the new features and compatibility. + +## Learn more + +For a complete list of all changes, see the +[full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.15.0). + +Check [the Install section](/install.html) to get Elixir installed and +read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +to learn more. + +Happy compiling! diff --git a/_posts/2023-06-22-type-system-updates-research-dev.markdown b/_posts/2023-06-22-type-system-updates-research-dev.markdown new file mode 100644 index 000000000..1765f7830 --- /dev/null +++ b/_posts/2023-06-22-type-system-updates-research-dev.markdown @@ -0,0 +1,85 @@ +--- +layout: post +title: "Type system updates: moving from research into development" +authors: +- José Valim +category: Announcements +excerpt: A short status update on the effort to bring a type system into Elixir. +--- + +A year ago, at ElixirConf EU 2022, we announced an effort to research +and develop a type system for Elixir ([video presentation](https://www.youtube.com/watch?v=Jf5Hsa1KOc8)) +([written report](/blog/2022/10/05/my-future-with-elixir-set-theoretic-types/)). + +This work is happening under the lead of [Giuseppe Castagna](https://www.irif.fr/~gc/), +CNRS Senior Researcher, and taken by +[Guillaume Duboc](https://www.irif.fr/users/gduboc/index) as part of his +PhD studies, with further guidance from myself (José Valim). + +This article is a summary of where we are in our efforts and where we +are going. + +## Out of research + +Our main goal during research is to find a type system that can model +most of Elixir's functional semantics and develop brand new theory on +the areas we found to be incompatible or lacking. We believe we were +able to achieve this goal with a gradual set-theoretic type system +and we are now ready to head towards development. Over the last 2 months, +we have published plenty of resources on our results: + + * [A technical report on the design principles of the Elixir type system](https://arxiv.org/abs/2306.06391) + * [A technical presentation by Guillaume Duboc at ElixirConf 2023 on the work above](https://youtube.com/watch?v=gJJH7a2J9O8) + * [An informal discussion with Giuseppe Castagna, Guillaume Duboc, and José Valim on the SmartLogic podcast](https://smartlogic.io/podcast/elixir-wizards/s10-e12-jose-guillaume-giuseppe-types-elixir/) + * [An informal Q&A with Guillaume Duboc, José Valim, and the community on Twitch](https://www.twitch.tv/videos/1841707383) + +Our focus so far has been on the semantics. While we have introduced a +new syntax capable of expressing the semantics of the new set-theoretic +type system, the syntax is not final as there are still no concrete +plans for user-facing changes to the language. Once we are confident +those changes will happen, we will have plenty of discussion with the +community about the type system interface and its syntax. + +The work so far has been made possible thanks to a partnership between +the [CNRS](https://www.cnrs.fr/fr) and [Remote](https://remote.com), +with sponsorships from [Fresha](https://www.fresha.com), +[Supabase](https://supabase.com), and [Dashbit](https://dashbit.co). + +## Into development + +While there is still on-going research, our focus for the second semester +of 2023 onwards is on development. + +Incorporating a type system into a language used at scale can be a daunting +task. Our concerns range from how the community will interact and use the +type system to how it will perform on large codebases. Therefore, our plan +is to gradually introduce our gradual (pun intended) type system into the +Elixir compiler. + +In the first release, types will be used just internally by the compiler. +The type system will extract type information from patterns and guards to +find the most obvious mistakes, such as typos in field names or type +mismatches from attempting to add an integer to a string, without introducing +any user-facing changes to the language. At this stage, our main goal is +to assess the performance impact of the type system and the quality of +the reports we can generate in case of typing violations. If we are +unhappy with the results, we still have time to reassess our work or drop +the initiative altogether. + +The second milestone is to introduce type annotations only in structs, +which are named and statically-defined in Elixir codebases. Elixir programs +frequently pattern match on structs, which reveals information about +the struct fields, but it knows nothing about their respective types. +By propagating types from structs and their fields throughout the program, +we will increase the type system’s ability to find errors while further +straining our type system implementation. + +The third milestone is to introduce the (most likely) `$`-prefixed type +annotations for functions, with no or very limited type reconstruction: +users can annotate their code with types, but any untyped parameter +will be assumed to be of the `dynamic()` type. If successful, then we +will effectively have introduced a type system into the language. + +This new exciting development stage is sponsored by [Fresha](https://www.fresha.com) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team) ([they are hiring!](https://starfish.team/jobs/experienced-elixir-developer)), +and [Dashbit](https://dashbit.co). diff --git a/_posts/2023-09-20-strong-arrows-gradual-typing.markdown b/_posts/2023-09-20-strong-arrows-gradual-typing.markdown new file mode 100644 index 000000000..cd62f8c33 --- /dev/null +++ b/_posts/2023-09-20-strong-arrows-gradual-typing.markdown @@ -0,0 +1,293 @@ +--- +layout: post +title: "Strong arrows: a new approach to gradual typing" +authors: +- José Valim +category: Announcements +excerpt: An introduction to strong arrows and how it leverages the Erlang VM to provide sound gradual typing. +--- + +*This is article expands on the topic of gradual set-theoretic typing discussed during my keynote at [ElixirConf US 2023](https://www.youtube.com/watch?v=giYbq4HmfGA).* + +There is an on-going effort [to research and develop a type system for Elixir](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/), lead by [Giuseppe Castagna](https://www.irif.fr/~gc/), CNRS Senior Researcher, and taken by [Guillaume Duboc](https://www.irif.fr/users/gduboc/index) as part of his PhD studies. + +In this article, we will discuss how the proposed type system will tackle gradual typing and how it relates to set-theoretic types, with the goal of providing an introduction to the ideas [presented in our paper](https://arxiv.org/abs/2306.06391). + +## Set-theoretic types + +The type system we are currently researching and developing for Elixir is based on set-theoretic types, which is to say its operations are based on the fundamental set operations of union, intersection, and negation. + +For example, the atom `:ok` is a value in Elixir, that can be represented by the type `:ok`. All atoms in Elixir are represented by themselves in the type system. A function that returns either `:ok` or `:error` is said to return `:ok or :error`, where the `or` operator represents the union. + +The types `:ok` and `:error` are contained by the type `atom()`, which is an infinite set representing all atoms. The union of the types `:ok` and `atom()` can be written as `:ok or atom()`, and is equivalent to `atom()` (as `:ok` is a subset of `atom()`). The intersection of the types `:ok` and `atom()` can be written as `:ok and atom()`, and is equivalent to `:ok`. + +Similarly, `integer()` is another infinite set representing all integers. `integer() or atom()` is the union of all integers and atoms. The intersection `integer() and atom()` is an empty set, which we call `none()`. The union of all types that exist in Elixir is called `term()`. + +The beauty of set-theoretic types is that we can model many interesting properties found in Elixir programs on top of those fundamental set operations, which in turn we hope to make typing in Elixir both more expressive and accessible. Let's see an example of how a type system feature, called bounded quantification (or bounded polymorphism), can be implemented with set-theoretic types. + +## Upper and lower bounds + +The `identity` function is a function that receives an argument and returns it as is. In Java, it would be written as follows: + +```java +static T identity(T arg) { + return arg; +} +``` + +In TypeScript: + +```typescript +function identity(arg: T): T { + return arg; +} +``` + +Or in Haskell: + +```haskell +id :: a -> a +id arg = arg +``` + +In all of the examples above, we say the function receives an argument of type variable `T` (or type variable `a` in Haskell's case) and return a value of the same type `T`. We call this parametric polymorphism, because the function parameter - its argument - can take many (poly) shapes (morphs). In Elixir, we could then support: + +```elixir +$ a -> a +def identity(arg), do: arg +``` + +Sometimes we may want to further constrain those type variables. As example, let's constraint the identity function in Java to numbers: + +```java +static T identity(T arg) { + return arg; +} +``` + +Or in TypeScript: + +```typescript +function identity(arg: T): T { + return arg; +} +``` + +In Haskell, we can constrain to a typeclass, such as `Ord`: + +```haskell +id :: Ord a => a -> a +id x = x +``` + +In other words, these functions can accept any type as long as they fulfill a given constraint. This in turn is called bounded polymorphism, because we are putting bounds on the types we can receive. + +With all that said, how can we implement bounded polymorphism in set-theoretic types? Imagine we have a type variable `a`, how can we ensure it is bounded or constrained to another type? + +With set-theoretic types, this operation is an intersection. If you have `a and atom()`, `a` can be the type `:foo`. `a` can also be the type `atom()`, which represents all atom types, but `a` cannot be `integer()`, as `integer() and atom()` will return an empty set. In other words, there is no need to introduce a new semantic construct, as intersections can be used to place upper bounds in type variables! Therefore, we could restrict Elixir's identity function to numbers like this: + +```elixir +$ a and number() -> a and number() +def identity(arg), do: arg +``` + +Of course, we can provide syntax sugar for those constraints: + +```elixir +$ a -> a when a: number() +def identity(arg), do: arg +``` + +But at the end of the day it will simply expand to intersections. The important bit is that, at the semantic level, there is no need for additional constructs and representations. + +> Note: for the type-curious readers, set-theoretic types implement [a limited form of bounded quantification *à la* Kernel Fun](http://lucacardelli.name/Papers/OnUnderstanding.pdf). In a nutshell, it means we can only compare functions if they have the same bounds. For example, our type system states `a -> a when a: integer() or boolean()` is not a subtype of `a -> a when a: integer()`. + +We also get lower bounds for free. If intersections allow us to place an upper bound on a type variable, a union is equivalent to a lower bound as it specifies the type variable will always be augmented by the union-ed type. For example, `a or atom()` says the result will always include atoms plus whatever else specified by `a` (which may be an atom, `atom()` itself, or a completely disjoint type such as `integer()`). + +Elixir protocols, which is an Elixir construct equivalent to Haskell Typeclasses or Java interfaces, is another example of functionality that can be modelled and composed with set-theoretic types without additional semantics. The exact mechanism to do so is left as an exercise to the reader (or the topic of a future blog post). + +## Enter gradual typing + +Elixir is a functional dynamic programming language. Existing Elixir programs are untyped, which means that a type system needs mechanisms to interface existing Elixir code with future statically typed Elixir code. We can achieve this with gradual typing. + +A gradual type system is a type system that defines a `dynamic()` type. It is sometimes written as `?` and sometimes known as the `any` type (but I prefer to avoid `any` because it is too short and too lax in languages like TypeScript). + +In Elixir, the `dynamic()` type means the type is only known at runtime, effectively disabling static checks for that type. More interestingly, we can also place upper and lower bounds on the dynamic type using set operations. As we will soon learn, this will reveal interesting properties about our type system. + +It is often said that gradual typing is the best of both words. Perhaps ironically, that's true and false at the same time. If you use a gradual type system but you never use the `dynamic()` type, then it behaves exactly like a static type system. However, the more you use the `dynamic()` type, the fewer guarantees the type system will give you, the more the `dynamic()` type propagates through the system. Therefore, it is in our interest to reduce the occurrences of the `dynamic()` type as much as possible, and that's what we set out to do. + +## Interfacing static and dynamic code: the trouble with `dynamic()` + +Let's go back to our constrained identity function that accepts only numbers: + +```elixir +$ a -> a when a: number() +def identity(arg), do: arg +``` + +Now imagine that we have some untyped code that calls this function: + +```elixir +def debug(arg) do + "we got: " <> identity(arg) +end +``` + +Since `debug/1` is untyped, its argument will receive the type `dynamic()`. + +`debug/1` proceeds to call `identity` with an argument and then uses the string concatenation operator (`<>`) to concatenate `"we got: "` to the result of `identity(arg)`. Since `identity/1` is meant to return a number and string concatenation requires two strings as operands, there is a typing error in this program. On the other hand, if you call `debug("hello")` at runtime, the code will work and won't raise any exceptions. + +In other words, the static typing version of the program and its runtime execution do not match in behaviour. So how do we tackle this? + +One option is to say that's all behaving as expected. If `debug/1` is untyped, its `arg` has the `dynamic()` type. To type check this program, we specify that `identity(dynamic())` returns the `dynamic()` type, the concatenation of a string with `dynamic()` also returns `dynamic()`, and consequently `debug/1` gets the type `dynamic() -> dynamic()`, with no type errors emitted. + +The trouble is: this is not a very useful choice. Once `dynamic()` enters the system, it _spreads everywhere_, we perform fewer checks, effectively discarding the information that `identity/1` returns a number, and the overall type system becomes less useful. + +Another option would be for us to say: once we call a statically typed function with `dynamic()`, we will ignore the `dynamic()` type. If the function says it returns a `number()`, then it will surely be a number! In this version, `identity(dynamic())` returns `number()` and the type system will catch a type error when concatenating a string with a number. + +This is similar to the approach taken by TypeScript. This means we can perform further static checks, but it also means we can call `debug("foobar")` and that will return the string `"we got: foobar"`! But how can that be possible when the type system told us that `identity` returns a `number()`? This can lead to confusion and surprising results at runtime. We say this system is unsound, because the types at runtime do not match our compile-time types. + +None of our solutions so far attempted to match the static and runtime behaviors, but rather, they picked one in favor of the other. + +But don't despair, there is yet another option. We could introduce runtime checks whenever we cross the "dynamic <-> static" boundaries. In this case, we could say `identity(dynamic())` returns a `number()`, but we will introduce a runtime check into the code to guarantee that's the case. This means we get static checks, we ensure the value is correct at runtime, with the cost of introducing additional checks at runtime. Unfortunately, those checks may affect performance, depending on the complexity of the data structure and on how many times we cross the "dynamic <-> static" boundary. + +> Note: there is [recent research in using the runtime checks introduced by a gradual type system to provide compiler optimizations](https://arxiv.org/abs/2206.13831). Some of these techniques are already leveraged by the Erlang VM to optimize code based on patterns and guards. + +To summarize, we have three options: + + * Calling static code from dynamic code returns `dynamic()`, dropping the opportunity of further static typing checks (this is sound) + + * Calling static code from dynamic code returns the static types, potentially leading to mismatched types at runtime (this is unsound) + + * Calling static code from dynamic code returns the static types with additional runtime checks, unifying both behaviours but potentially impacting performance (this is sound) + +## Introducing strong arrows + +I have always said that Elixir, thanks to Erlang, is an assertive language. For example, if our identity function is restricted to only numbers, in practice we would most likely write it as: + +```elixir +$ a -> a when a: number() +def identity(arg) when is_number(arg), do: arg +``` + +In the example above, `identity` will fail if given any value that is not a number. We often rely on pattern matching and guards and, in turn, they helps us assert on the types we are working with. Not only that, Erlang's JIT compiler already relies on this information to [perform optimizations](https://www.erlang.org/blog/type-based-optimizations-in-the-jit/) whenever possible. + +We also say Elixir is strongly typed because its functions and operators avoid implicit type conversions. The following functions also fail when their input does not match their type: + +```elixir +$ binary() -> binary() +def debug(string), do: "we got: " <> string + +$ (integer() -> integer()) and (float() -> float()) +def increment(number), do: number + 1 +``` + +`<>` only accepts binaries as arguments and will raise otherwise. `+` only accepts numbers (integers or floats) and will raise otherwise. `+` does not perform implicit conversions of non-numeric types, such as strings to number, as we can see next: + +```elixir +iex(1)> increment(1) +2 +iex(2)> increment(13.0) +14.0 +iex(3)> increment("foobar") +** (ArithmeticError) bad argument in arithmetic expression: "foobar" + 1 +``` + +In other words, Elixir's runtime consistently checks the values and their types at runtime. If `increment` fails when given something else than a number, then it will fail when the `dynamic()` type does not match its input at runtime. This guarantees `increment` returns its declared type and therefore we do not need to introduce runtime type checks when calling said function from untyped code. + +When we look at the `identity`, `debug`, and `increment` functions above, we - as developers - can state that these functions raise when given a value that does not match their input. However, how can we generalize this property so it is computed by the type system itself? To do so, we introduce a new concept called **strong arrows**, which relies on set-theoretical types to derive this property. + +The idea goes as follows: a strong arrow is a function that can be statically proven that, when evaluated on values outside of its input types (i.e. its domain), it will error. For example, in our `increment` function, if we pass a `string()` as argument, it won't type check, because `string() + integer()` is not a valid operation. Thanks to set-theoretic types, we can compute all values outside of the domain by computing the negation of a set. Given `increment/1` will fail for all types which are `not number()`, the function is strong. + +By applying this rule to all typed functions, we will know which functions are strong and which ones are not. If a function is strong, the type system knows that calling it with a `dynamic()` type will always evaluate to its return type! Therefore we say the return type of `increment(dynamic())` is `number()`, which is sound and does not need further runtime checks! + +Going back to our `debug` function, when used with a guarded identity, it will be able to emit warnings at compile-time, errors at runtime, without introducing any additional runtime check: + +```elixir +$ a -> a when a: number() +def identity(arg) when is_number(arg), do: arg + +def debug(arg) do + "we got: " <> identity(arg) +end +``` + +However, if the `identity` function is not strong, then we must fallback to one of the strategies in the previous section. + +Another powerful property of strong arrows is that they are composable. Let's pick an example from the paper: + +```elixir +$ number(), number() -> number() +def subtract(a, b) do + a + negate(b) +end + +$ number() -> number() +def negate(int), do: -int +``` + +In the example above, `negate/1`'s type is a strong arrow, as it raises for any input outside of its domain. `subtract/2`'s type is also a strong arrow, because both `+` and our own `negate` are strong arrows too. This is an important capability as it limits how `dynamic()` types spread throughout the system. + +> Errata: my presentation used the type `integer()` instead of `number()` for the example above. However, that was a mistake in the slide. Giving the type `integer(), integer() -> integer()` to `subtract` and `integer() -> integer()` to `negate` does not make `subtract` a strong arrow. Can you tell why? + +Luckily, other gradually typed languages can also leverage strong arrows. However, the more polymorphic a language and its functions are, the more unlikely it is to conclude that a given function is strong. For example, in other gradually typed languages such as Python or Ruby, the `+` operator is extensible and the user can define custom types where the operation is valid. In TypeScript, `"foobar" + 1` is also a valid operation, which expands the function domain. In both cases, an `increment` function restricted to numbers would not have a strong arrow type, as the operator won't fail for all types outside of `number()`. Therefore, to remain sound, they must either restrict the operands with further runtime checks or return `dynamic()` (reducing the number of compile-time checks). + +There is one last scenario to consider, which I did not include during my keynote for brevity. Take this function: + +```elixir +$ integer() -> :ok +def receives_integer_and_returns_ok(_arg), do: :ok +``` + +The function above can receive any type and return `:ok`. Is its type a strong arrow? Well, according to our definition, it is not. If we negate its input, type checking does not fail, it returns `:ok`. + +However, given the return type is always the same, it should be a strong arrow! To do so, let's amend and rephrase our definition of strong arrows: we negate the domain (i.e. the inputs) of a function and then type check it. If the function returns `none()` (i.e. it does not type check) or a type which is a subset of its codomain (i.e. its output), then it is a strong arrow. + +## Gradual typing and false positives + +There is one last scenario we must take into consideration when interfacing dynamic and static code. Imagine the following code: + +```elixir +def increment_and_remainder(numerator, denominator) do + rem(numerator, increment(denominator)) +end + +$ (integer() -> integer()) and (float() -> float()) +def increment(number), do: number + 1 +``` + +The `increment_and_remainder/2` function is untyped, therefore both of its arguments receive type `dynamic()`. The function then computes the remainder of the numerator by the denominator incremented by one. For this example, let's assume all uses of `increment_and_remainder/2` in our program passes two integers as arguments. + +Given `increment/1` has a strong arrow type, according to our definition, `increment(dynamic())` will return `integer() or float()` (also known as `number()`). Here lies the issue: if `increment(dynamic())` returns `integer() or float()`, the program above won't type check because `rem/2` does not accept floats. + +When faced with this problem, there are two possible reactions: + +1. It is correct for the function to not type check given `increment` may return a float + +2. It is incorrect for the function to not type check because the error it describes never occurs in the codebase + +Another interesting property of gradual set-theoretic types is that we can also place upper bounds on the `dynamic()` type. If a function returns `number()`, it means the caller needs to handle both `integer()` and `float()`. However, if a function returns `dynamic() and number()`, it means the type is defined at runtime, but it must still verify it is one of `integer()` or `float()` at compile time. + +Therefore, `rem/2` will type check if its second argument has the type `dynamic() and number()`, as there is one type at runtime (`integer()`) that satisfies type checking. On the other hand, if you attempt to use the string concatenation operator (`<>`) on `dynamic() and number()`, then there is no acceptable runtime type and you'd still get a typing violation! + +Going back to strong arrows, there are two possible return types from a strong arrow: + +1. A strong arrow, when presented with a dynamic type, returns its codomain + +2. A strong arrow, when presented with a dynamic type, returns the intersection of the codomain with the `dynamic()` type + +The second option opens up the possibility for existing codebases to gradually migrate to static types without dealing with false positives. Coming from a dynamic background, false positives can be seen as noisy or as an indication that static types are not worth the trouble. With strong arrows and gradual set-theoretic types, we will be able to explore different trade-offs on mixed codebases. Which of the two choices above we will adopt as a default and how to customize them is yet to be decided. It will depend on the community feedback as we experiment and integrate the type system. + +Erlang and Elixir developers who use Dialyzer will be familiar with these trade-offs, as the second option mirrors Dialyzer's behaviour of no false positives. The difference here is that our semantics are integrated into a complete type system. If no type signature is present, the `dynamic()` type is used, and we will leverage the techniques described here to interface dynamic and static code. If a function has a type signature, and no `dynamic()` type is present, then it will behave as statically typed code when called with statically typed arguments. Migrating to static types will naturally reduce the interaction points between dynamic and static code, removing the reliance on the `dynamic()` type. + +## Summary + +Set-theoretic types allow us to express many typing features based on set operations of union, intersection, and negation. + +In particular, we have been exploring a gradual set-theoretic type system for Elixir, paying special attention to how the type system will integrate with existing codebases and how it can best leverage the semantics of the Erlang Virtual Machine. The type system will also perform limited inference based on patterns and guards (as described in the paper), which - in addition to strong arrows - we hope to bring some of the benefits of static typing to codebases without changing a single line of code. + +While our efforts have officially moved from research into development, and [we have outlined an implementation plan](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/), we haven't yet fully implemented nor assessed the usability of set-theoretic types in existing Elixir codebases, either large or small. There is much to implement and validate, and we don't rule the possibility of finding unforeseen deal breakers that could send us back to square one. Yet we are pleased and cautiously excited with the new developments so far. + +The development of Elixir's type system is sponsored by [Fresha](https://www.fresha.com) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team) ([they are hiring!](https://starfish.team/jobs/experienced-elixir-developer)), +and [Dashbit](https://dashbit.co). diff --git a/_posts/2023-12-22-elixir-v1-16-0-released.markdown b/_posts/2023-12-22-elixir-v1-16-0-released.markdown new file mode 100644 index 000000000..6e4a61dca --- /dev/null +++ b/_posts/2023-12-22-elixir-v1-16-0-released.markdown @@ -0,0 +1,189 @@ +--- +layout: post +title: Elixir v1.16 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.16 released with compiler diagnostics and extensive documentation +--- + +Elixir v1.16 has just been released. 🎉 + +The Elixir team continues improving the developer experience +via tooling, documentation, and precise feedback, while keeping +the language stable and compatible. + +The notable improvements in this release are the addition of +compiler diagnostics and extensive improvements to our docs +in the forms of guides, anti-patterns, diagrams and more. + +## Code snippets in diagnostics + +Elixir v1.15 introduced a new compiler diagnostic format and +the ability to print multiple error diagnostics per compilation +(in addition to multiple warnings). + +With Elixir v1.16, we also include code snippets in exceptions +and diagnostics raised by the compiler, including ANSI coloring +on supported terminals. For example, a syntax error now includes +a pointer to where the error happened: + +``` +** (SyntaxError) invalid syntax found on lib/my_app.ex:1:17: + error: syntax error before: '*' + │ + 1 │ [1, 2, 3, 4, 5, *] + │ ^ + │ + └─ lib/my_app.ex:1:17 +``` + +For mismatched delimiters, it now shows both delimiters: + +``` +** (MismatchedDelimiterError) mismatched delimiter found on lib/my_app.ex:1:18: + error: unexpected token: ) + │ + 1 │ [1, 2, 3, 4, 5, 6) + │ │ └ mismatched closing delimiter (expected "]") + │ └ unclosed delimiter + │ + └─ lib/my_app.ex:1:18 +``` + +For unclosed delimiters, it now shows where the unclosed delimiter starts: + +``` +** (TokenMissingError) token missing on lib/my_app:8:23: + error: missing terminator: ) + │ + 1 │ my_numbers = (1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + ... + 8 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected ")") + │ + └─ lib/my_app:8:23 +``` + +Errors and warnings diagnostics also include code snippets. +When possible, we will show precise spans, such as on undefined variables: + +``` + error: undefined variable "unknown_var" + │ +5 │ a - unknown_var + │ ^^^^^^^^^^^ + │ + └─ lib/sample.ex:5:9: Sample.foo/1 +``` + +Otherwise the whole line is underlined: + +``` +error: function names should start with lowercase characters or underscore, invalid name CamelCase + │ +3 │ def CamelCase do + │ ^^^^^^^^^^^^^^^^ + │ + └─ lib/sample.ex:3 +``` + +A huge thank you to Vinícius Müller for working on the new diagnostics. + +## Revamped documentation + +The [ExDoc](https://github.com/elixir-lang/ex_doc) package provides Elixir developers +with one of the most complete and robust documentation generator. It [supports API +references, tutorials, cheatsheets, and more](/blog/2022/12/22/cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience/). + +However, because many of the language tutorials and reference documentation +were written before ExDoc, they were maintained separately as part of the +official website, separate from the language source code. With Elixir v1.16, +[we have moved our learning material to the language repository](https://hexdocs.pm/elixir/introduction.html). +This provides several benefits: + +1. Tutorials are versioned alongside their relevant Elixir version + +2. You get full-text search across all API reference and tutorials + +3. ExDoc will autolink module and function names in tutorials to their relevant API documentation + +Another feature we have incorporated in this release is the addition +of cheatsheets, starting with [a cheatsheet for the Enum module](https://hexdocs.pm/elixir/main/enum-cheat.html). +If you would like to contribute future cheatsheets to Elixir itself, +feel free to start a discussion and collect feedback on the +[Elixir Forum](https://elixirforum.com/). + +Finally, we have started enriching our documentation with +[Mermaid.js](https://mermaid.js.org/) diagrams. You can find examples +in the [GenServer](https://hexdocs.pm/elixir/GenServer.html) +and [Supervisor](https://hexdocs.pm/elixir/Supervisor.html) docs. + +Elixir has always been praised by its excellent documentation and +we are glad to continue to raise the bar for the whole ecosystem. + +## Living anti-patterns reference + +Elixir v1.16 incorporates and extends the work on [Understanding Code Smells +in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf), +by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/), +into [the official documention in the form of anti-patterns](https://hexdocs.pm/elixir/what-anti-patterns.html). +Our goal is to provide examples of potential pitfalls for library and +application developers, with additional context and guidance on how +to improve their codebases. + +In earlier versions, Elixir's official reference for library authors +included a list of anti-patterns for library developers. Lucas Vegi and +Marco Tulio Valente extended and refined this list based on the existing +literature, articles, and community input (including feedback based on +their prevalence in actual codebases). + +To incorporate the anti-patterns into the language, we trimmed the list down +to keep only anti-patterns which are unambiguous and actionable, and divided +them into four categories: [code-related](https://hexdocs.pm/elixir/code-anti-patterns.html), +[design-related](https://hexdocs.pm/elixir/design-anti-patterns.html), +[process-related](https://hexdocs.pm/elixir/process-anti-patterns.html), +and [meta-programming](https://hexdocs.pm/elixir/macro-anti-patterns.html). +Then we collected more community feedback during the release candidate +period, further refining and removing unclear guidance. + +We are quite happy with the current iteration of anti-patterns but +this is just the beginning. As they become available to the whole community, +we expect to receive more input, questions, and concerns. We will +continue listening and improving, as our ultimate goal is to provide +a live reference that reflects the practices of the ecosystem, +rather than a document that is written in stone and ultimately gets +out of date. A perfect example of this is [the recent addition of +"Sending unnecessary data" anti-pattern](https://github.com/elixir-lang/elixir/pull/13194), +which was contributed by the community and describes a pitfall that may +happen across codebases. + +## Type system updates + +As we get Elixir v1.16 out of door, the Elixir team will focus on bringing +the initial core for set-theoretic types into the Elixir compiler, with the +goal of running automated analysis in patterns and guards. This is [the first +step outlined in a previous article](/blog/2023/06/22/type-system-updates-research-dev/) +and is sponsored by [Fresha](https://www.fresha.com) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team) ([they are hiring!](https://starfish.team/jobs/experienced-elixir-developer)), +and [Dashbit](https://dashbit.co). + +## Learn more + +Other notable changes in this release are: + +* the addition of [`String.replace_invalid/2`](https://hexdocs.pm/elixir/String.html#replace_invalid/2), to help deal with invalid UTF-8 encoding + +* the addition of the `:limit` option in [`Task.yield_many/2`](https://hexdocs.pm/elixir/Task.html#yield_many/2) that limits the maximum number of tasks to yield + +* improved binary pattern matching by allowing prefix binary matches, such as `<<^prefix::binary, rest::binary>>` + +For a complete list of all changes, see the +[full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.16.0). + +Check [the Install section](/install.html) to get Elixir installed and +read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +to learn more. + +Happy learning! diff --git a/_posts/2024-03-05-veeps-elixir-case.markdown b/_posts/2024-03-05-veeps-elixir-case.markdown new file mode 100644 index 000000000..f0e60b7d1 --- /dev/null +++ b/_posts/2024-03-05-veeps-elixir-case.markdown @@ -0,0 +1,135 @@ +--- +layout: post +title: "Scaling a streaming service to hundreds of thousands of concurrent viewers at Veeps" +authors: +- Hugo Baraúna +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Veeps. +logo: /images/cases/logos/veeps.svg +tags: streaming scaling web +--- + +*Welcome to our series of [case studies about companies using Elixir in production](/cases.html).* + +[Veeps](https://veeps.com/) is a streaming service that offers direct access to live and on-demand events by award-winning artists at the most iconic venues. Founded in 2018, it became part of Live Nation Entertainment in 2021. + +Veeps have been named [one of the ten most innovative companies in music](https://www.fastcompany.com/90848907/most-innovative-companies-music-2023) and nominated for an Emmy. They currently hold the [Guinness World Record](https://www.guinnessworldrecords.com/world-records/650975-most-tickets-sold-for-a-livestreamed-concert-by-a-solo-male-artist-current-year) for the world's largest ticketed livestream by a solo male artist—a performance where Elixir and Phoenix played an important role in the backend during the streaming. + +This case study examines how Elixir drove Veeps' technical transformation, surpassing high-scale demands while keeping the development team engaged and productive. + + +## The challenge: scaling to hundreds of thousands of simultaneous users + +Imagine you are tasked with building a system that can livestream a music concert to hundreds of thousands of viewers around the world at the same time. + +In some cases, users must purchase a ticket before the concert can be accessed. For a famous artist, it’s not uncommon to see thousands of fans continuously refreshing their browsers and attempting to buy tickets within the first few minutes of the announcement. + +The Veeps engineering team needed to handle both challenges. + +Early on, the Veeps backend was implemented in [Ruby on Rails](https://rubyonrails.org/). Its first version could handle a few thousand simultaneous users watching a concert without any impact to stream quality, which was fine when you have a handful of shows but would be insufficient with the expected show load and massive increase in concurrent viewership across streams. It was around that time that [Vincent Franco](https://twitter.com/vinniefranco) joined Veeps as their CTO. + +Vincent had an extensive background in building and maintaining ticketing and event management software at scale. So, he used that experience to further improve the system to handle tens of thousands of concurrent users. However, it became clear that improving it to *hundreds* of thousands would be a difficult challenge, requiring substantial engineering efforts and increased operational costs. The team began evaluating other stacks that could provide the out-of-the-box tooling for scaling in order to reach both short and long-term goals. + + +## Adopting Elixir, hiring, and rewriting the system + +Vincent, who had successfully deployed Elixir as part of high-volume systems in the past, believed Elixir was an excellent fit for Veeps' requirements. + +Backed by his experience and several case studies from the Elixir community, such as [the one from Discord](/blog/2020/10/08/real-time-communication-at-scale-with-elixir-at-discord/), Vincent convinced management that Elixir could address their immediate scaling needs and become a reliable foundation on which the company could build. + +With buy-in from management, the plan was set in motion. They had two outstanding goals: + +* Prepare the platform to welcome the most famous artists in the world. +* Build their own team of engineers to help innovate and evolve the product. + +Vincent knew that hiring right-fit technical people can take time and he didn't want to rush the process. Hence, he hired [DockYard](https://dockyard.com/) to rebuild the system while simultaneously searching for the right candidates to build out the team. + +Eight months later, the system had been entirely rewritten in Elixir and Phoenix. Phoenix Channels were used to enrich the live concert experience, while Phoenix LiveView empowered the ticket shopping journey. + +The rewrite was put to the test shortly after with a livestream that remains one of Veeps’ biggest, still to this day. Before the rewrite, 20 Rails nodes were used during big events, whereas now, the same service requires only 2 Elixir nodes. And the new platform was able to handle 83x more concurrent users than the previous system. + +The increase in infrastructure efficiency significantly reduced the need for complex auto-scaling solutions while providing ample capacity to handle high-traffic spikes. + +> The rewrite marked the most extensive and intricate system migration in my career, and yet, it was also the smoothest. +> +> \- Vincent Franco, CTO + + +This was a big testament to Elixir and Phoenix's scalability and gave the team confidence that they made the right choice. + +By the time the migration was completed, Veeps had also assembled an incredibly talented team of two backend and two frontend engineers, which continued to expand and grow the product. + + +## Perceived benefits of using Elixir and its ecosystem + +After using Elixir for more than two years, Veeps has experienced significant benefits. Here are a few of them. + + +### Architectural simplicity + +Different parts of the Veeps system have different scalability requirements. For instance, when streaming a show, the backend receives metadata from users' devices every 30 seconds to track viewership. This is the so-called *Beaconing service*. + +Say you have 250,000 people watching a concert: the Beaconing service needs to handle thousands of requests per second for a few hours at a time. As a result, it needs to scale differently from other parts of the system, such as the merchandise e-commerce or backstage management. + +To tackle this issue, they built a distributed system. They packaged each subsystem as an [Elixir release](https://hexdocs.pm/elixir/config-and-releases.html#releases), totaling five releases. For the communication layer, they used distributed Erlang, which is built into Erlang/OTP, allowing seamless inter-process communication across networked nodes. + +In a nutshell, each node contains several processes with specific responsibilities. Each of these processes belongs to their respective [distributed process group](https://www.erlang.org/doc/man/pg.html). If node A needs billing information, it will reach out to any process within the "billing process group", which may be anywhere in the cluster. + +When deploying a new version of the system, they deploy a new cluster altogether, with all five subsystems at once. Given Elixir's scalability, the whole system uses 9 nodes, making a simple deployment strategy affordable and practical. As we will see, this approach is well-supported during development too, thanks to the use of Umbrella Projects. + + +### Service-oriented architecture within a monorepo + +Although they run a distributed system, they organize the code in only one repository, following the monorepo approach. To do that, they use the [Umbrella Project feature](https://hexdocs.pm/elixir/dependencies-and-umbrella-projects.html#content) from Mix, the build tool that ships with Elixir. + +Their umbrella project consists of 16 applications (at the time of writing), which they [sliced into five OTP releases](https://hexdocs.pm/mix/Mix.Tasks.Release.html#module-umbrellas). The remaining applications contain code that needs to be shared between multiple applications. For example, one of the shared applications defines all the structs sent as messages across the subsystems, guaranteeing that all subsystems use the same schemas for that exchanged data. + +> With umbrella projects, you can have the developer experience benefits of a single code repository, while being able to build a service-oriented architecture. +> +> \- Andrea Leopardi, Principal Engineer + + +### Reducing complexity with the Erlang/Elixir toolbox + +Veeps has an e-commerce platform that allows concert viewers to purchase artist merchandise. In e-commerce, a common concept is a shopping cart. Veeps associates each shopping cart as a [GenServer](https://hexdocs.pm/elixir/GenServer.html), which is a lightweight process managed by the Erlang VM. + +This decision made it easier for them to implement other business requirements, such as locking the cart during payments and shopping cart expiration. Since each cart is a process, the expiration is as simple as sending a message to a cart process based on a timer, which is easy to do using GenServers. + +For caching, the team relies on [ETS (Erlang Term Storage)](https://www.erlang.org/doc/man/ets.html), a high-performing key-value store part of the Erlang standard library. For cache busting between multiple parts of the distributed system, they use [Phoenix PubSub](https://github.com/phoenixframework/phoenix_pubsub), a real-time publisher/subscriber library that comes with [Phoenix](https://phoenixframework.org/). + +Before the rewrite, the Beaconing service used Google's Firebase. Now, the system uses [Broadway](https://elixir-broadway.org/) to ingest data from hundreds of thousands of HTTP requests from concurrent users. Broadway is an Elixir library for building concurrent data ingestion and processing pipelines. They utilized the library's capabilities to efficiently send requests to AWS services, regulating batch sizes to comply with AWS limits. They also used it to handle rate limiting to adhere to AWS service constraints. All of this was achieved with Broadway's built-in functionality. + +Finally, they use [Oban](https://getoban.pro/), an Elixir library for background jobs, for all sorts of background-work use cases. + +Throughout the development journey, Veeps consistently found that Elixir and its ecosystem had built-in solutions for their technical challenges. Here's what Vincent, CTO of Veeps, had to say about that: + +> Throughout my career, I've worked with large-scale systems at several companies. However, at Veeps, it's unique because we achieve this scale with minimal reliance on external tools. It's primarily just Elixir and its ecosystem that empower us. +> +> \- Vincent Franco, CTO + +This operational simplicity benefitted not only the production environment but also the development side. The team could focus on learning Elixir and its ecosystem without the need to master additional technologies, resulting in increased productivity. + + +### LiveView: simplifying the interaction between front-end and back-end developers + +After the rewrite, [LiveView](https://github.com/phoenixframework/phoenix_live_view), a Phoenix library for building interactive, real-time web apps, was used for every part of the front-end except for the "Onstage" subsystem (responsible for the live stream itself). + +The two front-end developers, who came from a React background, also started writing LiveView. After this new experience, the team found the process of API negotiation between the front-end and back-end engineers much simpler compared to when using React. This was because they only had to use Elixir modules and functions instead of creating additional HTTP API endpoints and all the extra work that comes with them, such as API versioning. + +> Our front-end team, originally proficient in React, has made a remarkable transition to LiveView. They've wholeheartedly embraced its user-friendly nature and its smooth integration into our system. +> +> \- Vincent Franco, CTO + + +## Conclusion: insights from Veeps' Elixir experience + +The decision to use Elixir has paid dividends beyond just system scalability. The team, with varied backgrounds in Java, PHP, Ruby, Python, and Javascript, found Elixir's ecosystem to be a harmonious balance of simplicity and power. + +By embracing Elixir's ecosystem, including Erlang/OTP, Phoenix, LiveView, and Broadway, they built a robust system, eliminated the need for numerous external dependencies, and kept productively developing new features. + + +> Throughout my career, I've never encountered a developer experience as exceptional as this. Whether it's about quantity or complexity, tasks seem to flow effortlessly. The team's morale is soaring, everyone is satisfied, and there's an unmistakable atmosphere of positivity. We're all unequivocally enthusiastic about this language. +> +> \- Vincent Franco, CTO + +Veeps' case illustrates how Elixir effectively handles high-scale challenges while keeping the development process straightforward and developer-friendly. diff --git a/_posts/2024-06-12-elixir-v1-17-0-released.markdown b/_posts/2024-06-12-elixir-v1-17-0-released.markdown new file mode 100644 index 000000000..c857c906e --- /dev/null +++ b/_posts/2024-06-12-elixir-v1-17-0-released.markdown @@ -0,0 +1,178 @@ +--- +layout: post +title: "Elixir v1.17 released: set-theoretic data types, calendar durations, and Erlang/OTP 27 support" +authors: +- Andrea Leopardi +category: Releases +excerpt: "Elixir v1.17 released: set-theoretic data types, calendar durations, and Erlang/OTP 27 support" +--- + +Elixir v1.17 has just been released. 🎉 + +This release introduces set-theoretic types into a handful of language constructs. While there are still [many steps ahead of us](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/), this important milestone already brings benefits to developers in the form of new warnings for common mistakes. This new version also adds support for [Erlang/OTP 27](https://www.erlang.org/downloads/27), the latest and greatest Erlang release. You'll also find a new calendar-related data type (`Duration`) and a `Date.shift/2` function. + +Let's dive in. + +## Warnings from gradual set-theoretic types + +This release introduces gradual set-theoretic types to infer types from patterns and use them to type check programs, enabling the Elixir compiler to find faults and bugs in codebases without requiring changes to existing software. The underlying principles, theory, and roadmap of our work have been outlined in ["The Design Principles of the Elixir Type System" by Giuseppe Castagna, Guillaume Duboc, José Valim](https://arxiv.org/abs/2306.06391). + +At the moment, Elixir developers will interact with set-theoretic types only through **warnings** found by the type system. The current implementation models all data types in the language: + + * `binary()`, `integer()`, `float()`, `pid()`, `port()`, `reference()` - these + types are indivisible. This means both `1` and `13` get the same `integer()` + type. + + * `atom()` - it represents all atoms and it is divisible. For instance, the + atom `:foo` and `:hello_world` are also valid (distinct) types. + + * `map()` and structs - maps can be "closed" or "open". Closed maps only allow + the specified keys, such as `%{key: atom(), value: integer()}`. Open maps + support any other keys in addition to the ones listed and their definition + starts with `...`, such as `%{..., key: atom(), value: integer()}`. Structs + are closed maps with the `__struct__` key. + + * `tuple()`, `list()`, and `function()` - currently they are modelled as + indivisible types. The next Elixir versions will also introduce fine-grained + support to them. + +We focused on *atoms* and *maps* on this initial release as they are respectively the simplest and the most complex types representations, so we can stress the performance of the type system and quality of error messages. Modelling these types will also provide the most immediate benefits to Elixir developers. Assuming there is a variable named `user`, holding a `%User{}` struct with a `address` field, Elixir v1.17 will emit the following warnings at compile-time: + + * Pattern matching against a map or a struct that does not have the given key, + such as `%{adress: ...} = user` (notice `address` vs `adress`). + + * Accessing a key on a map or a struct that does not have the given key, such + as `user.adress`. + + * Invoking a function on non-modules, such as `user.address()`. + + * Capturing a function on non-modules, such as `&user.address/0`. + + * Attempting to call an anonymous function without an actual function, such as + `user.()`. + + * Performing structural comparisons between structs, such as `my_date < + ~D[2010-04-17]`. + + * Performing structural comparisons between non-overlapping types, such as + `integer >= string`. + + * Building and pattern matching on binaries without the relevant specifiers, + such as `<>` (this warns because by default it expects an integer, it + should have been `<>` instead). + + * Attempting to rescue an undefined exception or a struct that is not an + exception. + + * Accessing a field that is not defined in a rescued exception. + +Here's an example of how the warning for accessing a misspelled field of a +struct looks like: + +![Example of a warning when accessing a mispelled struct field](/images/contents/type-warning-on-struct-field.png) + +Another example, this time it's a warning for structural comparison across two +`Date` structs: + +![Example of a warning when comparing two structs with ">"](/images/contents/type-warning-on-date-comparison.png) + +These warnings also work natively in text editors, as they are standard Elixir +compiler warnings: + +![Example of a type warning inline in an editor](/images/contents/type-warning-in-editor.png) + +These new warnings will help Elixir developers find bugs earlier and give more +confidence when refactoring code, especially around maps and structs. While +Elixir already emitted some of these warnings in the past, those were discovered +using syntax analysis. The new warnings are more reliable, precise, and with +better error messages. Keep in mind, however, that the Elixir typechecker only +infers types from patterns within the same function at the moment. Analysis from +guards and across function boundaries will be added in future releases. For more +details, see our new [reference document on gradual set-theoretic +types](https://hexdocs.pm/elixir/gradual-set-theoretic-types.html). + +The type system was made possible thanks to a partnership between +[CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development +work is currently sponsored by [Fresha](https://www.fresha.com/) +([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +## Erlang/OTP support + +This release adds support for Erlang/OTP 27 and drops support for Erlang/OTP 24. +We recommend Elixir developers to migrate to Erlang/OTP 26 or later, especially +on Windows. Support for WERL (a graphical user interface for the Erlang terminal +on Windows) will be removed in Elixir v1.18. + +You can read more about Erlang/OTP 27 in [their release +announcement](https://www.erlang.org/downloads/27). The bits that are +particularly interesting for Elixir developers are the addition of a [`json` +module](https://erlang.org/documentation/doc-15.0-rc3/lib/stdlib-6.0/doc/html/json.html) +and process labels (`proc_lib:set_label/1`). The latter will also be available +in this Elixir release as `Process.set_label/1`. + +## New `Duration` data type and shifting functions + +This Elixir version introduces the `Duration` data type and APIs to shift dates, +times, and date times by a given duration, considering different calendars and +time zones. + +```elixir +iex> Date.shift(~D[2016-01-31], month: 2) +~D[2016-03-31] +``` + +We chose the name *"shift"* for this operation (instead of "add") since working +with durations does not obey properties such as **associativity**. For instance, +adding one month and then one month does not give the same result as adding two +months: + +```elixir +iex> ~D[2016-01-31] |> Date.shift(month: 1) |> Date.shift(month: 1) +~D[2016-03-29] +``` + +Still, durations are essential for building intervals, recurring events, and +modelling scheduling complexities found in the world around us. For `DateTime`s, +Elixir will correctly deal with time zone changes (such as Daylight Saving +Time). However, provisions are also available in case you want to surface +conflicts, such as shifting to a wall clock that does not exist, because the +clock has been moved forward by one hour. See `DateTime.shift/2` for examples. + +Finally, we added a new `Kernel.to_timeout/1` function, which helps developers +normalize durations and integers to a timeout used by many APIs—like `Process`, +`GenServer`, and more. For example, to send a message after one hour, you can +now write: + +```elixir +Process.send_after(pid, :wake_up, to_timeout(hour: 1)) +``` + +## Learn more + +Here are other notable changes in this release: + + * There are new `Keyword.intersect/2,3` functions to mirror the equivalent in + the `Map` module. + + * A new Mix profiler was added, `mix profile.tprof`, which lets you use the + new [tprof](https://www.erlang.org/doc/apps/tools/tprof.html) + profiler released with Erlang/OTP 27. This profiler leads to the + soft-deprecation of `mix profile.cprof` and `mix profile.eprof`. + + * We added `Kernel.is_non_struct_map/1`, a new guard to help with the common + pitfall of matching on `%{}`, which also successfully matches structs (as + they are maps underneath). + + * Elixir's Logger now formats + [`gen_statem`](https://www.erlang.org/doc/apps/stdlib/gen_statem.html) + reports and includes Erlang/OTP 27 *process labels* in logger events. + +For a complete list of all changes, see the +[full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.17.0). + +Check [the Install section](/install.html) to get Elixir installed and +read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +to learn more. + +Happy learning! diff --git a/_posts/2024-08-15-welcome-elixir-language-server-team.markdown b/_posts/2024-08-15-welcome-elixir-language-server-team.markdown new file mode 100644 index 000000000..b9e29ca0f --- /dev/null +++ b/_posts/2024-08-15-welcome-elixir-language-server-team.markdown @@ -0,0 +1,91 @@ +--- +layout: post +title: "Announcing the official Elixir Language Server team" +authors: +- José Valim +category: Announcements +excerpt: "Announcing the official Elixir Language Server team to work on the code intelligence infrastructure to be used across tools and editors" +--- + +> Update: [the new language server is called Expert and is currently available in alpha as open source](https://github.com/elixir-lang/expert). Companies wanting to directly sponsor work on Expert, please reach out to [Dan Janowski](mailto:sponsor+expert@erlef.org) from the [Erlang Ecosystem Foundation](https://erlef.org), which has kindly stepped in to manage and bring visibility into the project sponsorships. + +I am glad to welcome Elixir's official Language Server team, formed by (in alphabetical order): + +* [Jonatan Kłosko](https://github.com/jonatanklosko) +* [Łukasz Samson](https://github.com/lukaszsamson) +* [Mitch Hanberg](https://www.mitchellhanberg.com/) +* [Steve Cohen](https://github.com/scohen) + +The team will work on the code intelligence infrastructure to be used across tools and editors. These efforts are partially funded by [Fly.io](https://fly.io/) and [Livebook](https://livebook.dev/). + +## A brief history + +The [Language Server Protocol (LSP)](https://en.wikipedia.org/wiki/Language_Server_Protocol) was created by Microsoft as a protocol between IDEs and programming languages to provide language intelligence tools. + +The first implementation of said protocol for Elixir was started by [Jake Becker](https://github.com/JakeBecker/elixir-ls/), back in 2017, alongside an implementation for Visual Studio Code, and it relied on [the ElixirSense project from Marlus Saraiva](https://github.com/msaraiva/elixir_sense) to extract and provide some of the language intelligence. + +As the Language Server Protocol adoption grew as a whole, so did the usage of Elixir's implementation, which eventually became the main mechanism Elixir users interacted with the language from their editors. + +Eventually, Elixir's language server implementation got its [own organization on GitHub](https://github.com/elixir-lsp/), and maintenance reins were given to Łukasz Samson and Jason Axelson. + +Over time, the Elixir Language Server has accrued technical debt. Some of it exists due to intrinsic complexities (for example, the Language Server Protocol uses UTF-16 for text encoding, instead of the more widely used UTF-8), while others are a consequence of working on codebase while both the programming language and the protocol themselves were still evolving. + +This led to Mitch Hanberg and Steve Cohen to create alternative language server implementations, exploring different trade-offs. + +For example, both [Next LS](https://github.com/elixir-tools/next-ls) and [Lexical](https://github.com/lexical-lsp/lexical) use Erlang Distribution to isolate the Language Server runtime from the user code. + +Next LS also focused on extracting the LSP protocol parts into [GenLSP](https://github.com/elixir-tools/gen_lsp) (which can be used by anyone to easily create a language server), single binary distribution with [Burrito](https://github.com/burrito-elixir/burrito), and experimenting with SQLite for the symbol index. + +[Lexical](https://github.com/lexical-lsp/lexical) concerned itself with speed and abstractions to deal with documents, ranges, and more. + +This means the Elixir community had, for some time, three distinct language server implementations, each with their own strengths. + +## Looking forward + +The current language server maintainers have agreed to move forward with a _single Language Server Protocol project_, relying on the strengths of each implementation: + +* Lexical provides a stable foundation +* ElixirLS, through ElixirSense, provides the most complete implementation and wider functionality +* Next LS, through GenLSP, provides a general interface for LSP implementations and straight-forward packaging via [Burrito](https://github.com/burrito-elixir/burrito) + +The above is a rough outline, as the specific details of how the projects will move forward are still being discussed. While some of the team members also maintain direct integration with some editors, we will continue relying on the community's help and efforts to get full coverage across all available editors. + +And there is still a lot more to do! + +Many underestimate the complexity behind implementing the Language Server Protocol. That's not surprising: we mostly interact with it from an editor, allowing us to freely ignore what makes it tick. + +In practice, the Language Server needs, in many ways, to reimplement several parts of the language and its compiler. + +If the Elixir compiler sees the code `some_value +`, it can immediately warn and say: "this expression is incomplete". However, the Language Server still needs to make sense of invalid code to provide features like completion. And that applies to everything: missing do-end blocks, invalid operators, invoking macros that do not exist, etc. Mitch has made [Spitfire](https://github.com/elixir-tools/spitfire), an error tolerant parser to tackle this particular problem. + +Some ecosystems have undertaken [multi-year efforts to redesign their compilers and toolchains](https://en.wikipedia.org/wiki/Roslyn_(compiler)) to provide better tools for lexical and semantic code analysis (which most likely took a significant investment of time and resources to conclude). That's to say some of the problems faced by Language Server implementations will be best tackled if they are also solved as part of Elixir itself. + +For example, every Language Server implementation compiles their own version of a project, making it so every application and its dependencies have to be compiled twice in development: once for Mix and once for the Language Server. Wouldn't it be nice if Elixir and the Language Servers could all rely on the same compilation artifacts? + +This is not news to the Elixir team either: almost every Elixir release within the last 3 years has shipped new code analysis APIs, such as [Code.Fragment](https://hexdocs.pm/elixir/Code.Fragment.html), with the goal of removing duplication across Language Servers, [IEx](https://hexdocs.pm/iex), and [Livebook](https://livebook.dev/), as well as reduce their reliance on internal Elixir modules. Most recently, Elixir v1.17 shipped with [new APIs to help developers emulate the compiler behaviour](https://hexdocs.pm/elixir/Macro.Env.html). Our goal is to make these building blocks available for all Elixir developers, so their benefits are reaped beyond the language server tooling. + +Furthermore, as [set-theoretic types make their way into Elixir](https://elixir-lang.org/blog/2024/06/12/elixir-v1-17-0-released/), we also want to provide official APIs to integrate them into our tools. + +## Sponsorships + +Currently, [Fly.io](https://fly.io/) is sponsoring Łukasz Samson to work part-time on the Language Server and editor integration. The [Livebook](https://livebook.dev/) project is donating development time from Jonatan Kłosko, creator of Livebook, to improve the Elixir compiler and its code intelligence APIs. + +We are grateful to both companies for investing into the community and you should check them out. + +As mentioned above, Language Server implementations are complex projects, and unifying efforts is an important step in the right direction. However, we also need community help, and one of the ways to do so is by sponsoring the developers making this possible: + +* [Łukasz Samson](https://github.com/sponsors/lukaszsamson) +* [Mitch Hanberg](https://github.com/sponsors/mhanberg) +* [Steve Cohen](https://github.com/sponsors/scohen) + +Companies who can afford to sponsor part-time development are welcome to reach out and help us achieve this important milestone. + +## Progress updates + +A new project website and social media accounts will be created soon, and you can follow them to stay up to date with our progress and any interesting developments. + +The name of the new project is still in the works as well as many of the decisions we'll need to make, so please have patience! + +In the meantime, you can continue to use the language server of your choice, and we’ll be sure to make the transition to the fourth and final project as smooth as possible. + +Thank you! diff --git a/_posts/2024-08-28-typing-lists-and-tuples.markdown b/_posts/2024-08-28-typing-lists-and-tuples.markdown new file mode 100644 index 000000000..545996655 --- /dev/null +++ b/_posts/2024-08-28-typing-lists-and-tuples.markdown @@ -0,0 +1,173 @@ +--- +layout: post +title: "Typing lists and tuples in Elixir" +authors: +- José Valim +category: Internals +excerpt: "This article explores the design decisions of typing lists and tuples in Elixir within a sound gradual type system" +--- + +We have been working on [a type system for the Elixir programming language](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/). The type system provides sound gradual typing: it can safely interface static and dynamic code, and if the program type checks, it will not produce type errors at runtime. + +It is important to emphasize **type errors**. The type systems used at scale today do not guarantee the absense of any runtime errors, but only typing ones. Many programming languages error when accessing the "head" of an empty list, most languages raise on division by zero or when computing the logarithm of negative numbers on a real domain, and others may fail to allocate memory or when a number overflows/underflows. + +Language designers and maintainers must outline the boundaries of what can be represented as typing errors and how that impacts the design of libraries. The goal of this article is to highlight some of these decisions in the context of lists and tuples in Elixir's on-going type system work. + +> In this article, the words "raise" and "exceptions" describe something unexpected happened, and not a mechanism for control-flow. Other programming languages may call them "panics" or "faults". + +## The `head` of a list + +Imagine you are designing a programming language and you want to provide a `head` function, which returns the head - the first element - of a list, you may consider three options. + +The first option, the one found in many programming languages, is to raise if an empty list is given. Its implementation in Elixir would be something akin to: + +```elixir +$ list(a) -> a +def head([head | _]), do: head +def head([]), do: raise "empty list" +``` + +Because the type system cannot differentiate between an empty list and a non-empty list, you won't find any typing violations at compile-time, but an error is raised at runtime for empty lists. + +An alternative would be to return an `option` type, properly encoding that the function may fail (or not): + +```elixir +$ list(a) -> option(a) +def head([head | _]), do: {:ok, head} +def head([]), do: :none +``` + +This approach may be a bit redundant. Returning an `option` type basically forces the caller to pattern match on the returned `option`. While many programming languages provide functions to compose `option` values, one may also get rid of the additional wrapping and directly pattern match on the list instead. So instead of: + +```elixir +case head(list) do + {:ok, head} -> # there is a head + :none -> # do what you need to do +end +``` + +You could just write: + +```elixir +case list do + [head | _] -> # there is a head + [] -> # do what you need to do +end +``` + +Both examples above are limited by the fact the type system cannot distinguish between empty and non-empty lists and therefore their handling must happen at runtime. If we get rid of this limitations, we could define `head` as follows: + +```elixir +$ non_empty_list(a) -> a +def head([head | _]), do: head +``` + +And now we get a typing violation at compile-time if an empty list is given as argument. There is no `option` tagging and no runtime exceptions. Win-win? + +The trouble with the above is that now it is responsibility of the language users to prove the list is not empty. For example, imagine this code: + +```elixir +list = convert_json_array_to_elixir_list(json_array_as_string) +head(list) +``` + +In the example above, since `convert_json_array_to_elixir_list` may return an empty list, there is a typing violation at compile-time. To resolve it, we need to prove the result of `convert_json_array_to_elixir_list` is not an empty list before calling `head`: + +```elixir +list = convert_json_array_to_elixir_list(json_array_as_string) + +if list == [] do + raise "empty list" +end + +head(list) +``` + +But, at this point, we might as well just use pattern matching and once again get rid of `head`: + +```elixir +case convert_json_array_to_elixir_list(json_array_as_string) do + [head | _] -> # there is a head + [] -> # do what you need to do +end +``` + +Most people would expect that encoding more information into the type system would bring only benefits but there is a tension here: the more you encode into types, the more you might have to prove in your programs. + +While different developers will prefer certain idioms over others, I am not convinced there is one clearly superior approach here. Having `head` raise a runtime error may be the most pragmatic approach _if_ the developer expects the list to be non-empty in the first place. Returning `option` gets rid of the exception by forcing users to explicitly handle the result, but leads to more boilerplate compared to pattern matching, especially if the user does not expect empty lists. And, finally, adding precise types means there could be more for developers to prove. + +### What about Elixir? + +Thanks to set-theoretic types, we will most likely distinguish between empty lists and non-empty lists in Elixir's type system, since pattern matching on them is a common language idiom. Furthermore, several functions in Elixir, such as `String.split/2` are guaranteed to return non-empty lists, which can then be nicely encoded into a function's return type. + +Elixir also has the functions `hd` (for head) and `tl` (for tail) inherited from Erlang, which are [valid guards](https://hexdocs.pm/elixir/patterns-and-guards.html). They only accept non-empty lists as arguments, which will now be enforced by the type system too. + +This covers almost all use cases but one: what happens if you want to access the first element of a list, which has not been proven to be empty? You could use pattern matching and conditionals for those cases, but as seen above, this can lead to common boilerplate such as: + +```elixir +if list == [] do + raise "unexpected empty list" +end +``` + +Luckily, it is common in Elixir to use the `!` suffix to encode the possibility of runtime errors for _valid_ inputs. For these circumstances, we may introduce `List.first!` (and potentially `List.drop_first!` for the tail variant). + +## Accessing tuples + +Now that we have discussed lists, we can talk about tuples. In a way, tuples are more challenging than lists for two reasons: + +1. A list is a collection where all elements have the same type (be it a `list(integer())` or `list(integer() or float())`), while tuples carry the types of each element + +2. We natively access tuples by index, instead of its head and tail, such `elem(tuple, 0)` + +In the upcoming v1.18 release, Elixir's new type system will support tuple types, and they are written between curly brackets. For example, the [`File.read/1` function](https://hexdocs.pm/elixir/File.html#read/1) would have the return type `{:ok, binary()} or {:error, posix()}`, quite similar to today's typespecs. + +The tuple type can also specify a minimum size, as you can also write: `{atom(), integer(), ...} `. This means the tuple has at least two elements, the first being an `atom()` and the second being an `integer()`. This definition is required for type inference in patterns and guards. After all, a guard `is_integer(elem(tuple, 1))` tells you the tuple has at least two elements, with the second one being an integer, but nothing about the other elements and the tuple overall size. + +With tuples support merged into main, we need to answer questions such as which kind of compile-time warnings and runtime exceptions tuple operations, such as `elem(tuple, index)` may emit. Today, we know that it raises if: + +1. the index is out of bounds, as in `elem({:ok, "hello"}, 3)` + +2. the index is negative, as in `elem({:ok, 123}, -1)` + +When typing `elem(tuple, index)`, one option is to use "avoid all runtime errors" as our guiding light and make `elem` return `option` types, such as: `{:ok, value}` or `:none`. This makes sense for an out of bounds error, but should it also return `:none` if the index is negative? One could argue that they are both out of bounds. On the other hand, a positive index may be correct depending on the tuple size but **a negative index is always invalid**. From this perspective, encoding an always invalid value as an `:none` can be detrimental to the developer experience, hiding logical bugs instead of (loudly) blowing up. + +Another option is to make these programs invalid. If we completely remove `elem/2` from the language and you can only access tuples via pattern matching (or by adding a literal notation such as `tuple.0`), then all possible bugs can be caught by the type checker. However, some data structures, such as [array in Erlang](https://www.erlang.org/doc/apps/stdlib/array.html) rely on dynamic tuple access, and implementing those would be no longer possible. + +Yet another option is to encode integers themselves as values in the type system. In the same way that Elixir's type system supports the values `:ok` and `:error` as types, we could support each integer, such as `13` and `-42` as types as well (or specific subsets, such as `neg_integer()`, `zero()` and `pos_integer()`). This way, the type system would know the possible values of `index` during type checking, allowing us to pass complex expressions to `elem(tuple, index)`, and emit typing errors if the indexes are invalid. However, remember that encoding more information into types may force developers to also prove that those indexes are within bounds in many other cases. + +Once again, there are different trade-offs, and we must select one that best fit into Elixir use and semantics today. + +### What about Elixir? + +The approach we are taking in Elixir is two-fold: + +* If the index is a literal integer, it will perform an exact access on the tuple element. This means `elem(tuple, 1)` will work if we can prove the tuple has at least size 2, otherwise you will have a type error + +* If the index is not a literal integer, the function will fallback to a dynamic type signature + +Let's expand on the second point. + +At a fundamental level, we could describe `elem` with the type signature of `tuple(a), integer() -> a`. However, the trouble with this signature is that it does not tell the type system (nor users) the possibility of a runtime error. Luckily, because Elixir will offer a gradual type system, we could encode the type signature as `dynamic({...a}), integer() -> dynamic(a)`. By encoding the argument and return type as dynamic, developers who want a fully static program will be notified of a typing error, while existing developers who rely on dynamic features of the language can continue to do so, and those choices are now encoded into the types. + +Overall, + +* For static programs (the ones that do not use the `dynamic()` type), `elem/2` will validate that the first argument is a tuple of known shape, and the second argument is a literal integer which is greater than or equal to zero and less than the tuple size. This guarantees no runtime exceptions. + +* Gradual programs will have the same semantics (and runtime exceptions) as today. + +## Summary + +I hope this article outlines some of the design decisions as we bring a gradual type system to Elixir. Although supporting tuples and lists is a "table stakes" feature in most type systems, bringing them to Elixir is an opportunity to understand how the type system will interact with several language idioms, as well as provide a foundation for future decisions. The most important take aways are: + +1. Type safety is a commitment from both sides. If you want your type system to find even more bugs through more precise types, you will need to prove more frequently that your programs are free of certain typing violations. + +2. It is not a goal of the type system to avoid all runtime errors. This would require either a type system that is too precise (and require more proofs) or it would require functions to mask hard errors (such as a negative index) as error values. Exceptions still play an important role in typed Elixir and, given they are modelled as structs, they will also be typed in the future. + +3. Elixir's convention of using the suffix `!` to provide variants that encode the possibility of runtime exceptions for a valid domain (the input types) nicely complements the type system, as it can help static programs avoid the boilerplate of converting `:none`/`:error` into exceptions for unexpected scenarios. + +4. Using `dynamic()` in function signatures is a mechanism available in Elixir's type system to signal that a function has dynamic behaviour and may raise runtime errors, allowing violations to be reported on programs that wish to remain fully static. Similar to how other static languages provide dynamic behaviour via `Any` or `Dynamic` types. + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), [Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +Happy typing! diff --git a/_posts/2024-12-19-elixir-v1-18-0-released.markdown b/_posts/2024-12-19-elixir-v1-18-0-released.markdown new file mode 100644 index 000000000..22edc0364 --- /dev/null +++ b/_posts/2024-12-19-elixir-v1-18-0-released.markdown @@ -0,0 +1,218 @@ +--- +layout: post +title: "Elixir v1.18 released: type checking of calls, LSP listeners, built-in JSON, and more" +authors: +- José Valim +category: Releases +excerpt: "Elixir v1.18 released: type checking of function calls, Language Server listeners, built-in JSON, ExUnit improvements, mix format --migrate, and more" +--- + +Elixir v1.18 is an impressive release with improvements across the two main efforts happening within the Elixir ecosystem right now: set-theoretic types and language servers. It also comes with built-in JSON support and adds new capabilities to its unit testing library. Let's go over each of those in detail. + +## Type inference of patterns and return types + +There are several updates in the typing department, so let's break them down. + +#### A type system? In my Elixir? + +There is an on-going [research and development](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/) effort to bring static types to Elixir. Elixir's type system is: + + * **sound** - the types inferred and assigned by the type system align with the behaviour of the program + + * **gradual** - Elixir's type system includes the `dynamic()` type, which can be used when the type of a variable or expression is checked at runtime. In the absence of `dynamic()`, Elixir's type system behaves as a static one + + * **developer friendly** - the types are described, implemented, and composed using basic set operations: unions, intersections, and negation (hence it is a set-theoretic type system) + +More interestingly, you can compose `dynamic()` with any type. For example, `dynamic(integer() or float())` means the type is either `integer()` or `float()` at runtime. This allows the type system to emit warnings if none of the types are satisfied, even in the presence of dynamism. + +#### What has already been done? + +[Elixir v1.17 was the first release to incorporate the type system in the compiler](https://elixir-lang.org/blog/2024/06/12/elixir-v1-17-0-released/). In particular, we have added support for primitive types (integer, float, binary, pids, references, ports), atoms, and maps. We also added type checking to a handful of operations related to those types, such as accessing fields in maps, as in `user.adress` (mind the typo), performing structural comparisons between structs, as in `my_date < ~D[2010-04-17]`, etc. + +#### What is new in v1.18? + +The most exciting change in Elixir v1.18 is type checking of function calls, alongside gradual inference of patterns and return types. To understand how this will impact your programs, consider the following code defined in `lib/user.ex`: + +```elixir +defmodule User do + defstruct [:age, :car_choice] + + def drive(%User{age: age, car_choice: car}, car_choices) when age >= 18 do + if car in car_choices do + {:ok, car} + else + {:error, :no_choice} + end + end + + def drive(%User{}, _car_choices) do + {:error, :not_allowed} + end +end +``` + +Elixir's type system will infer the `drive` function expects a `User` struct as input and returns either `{:ok, dynamic()}` or `{:error, :no_choice}` or `{:error, :not_allowed}`. Therefore, the following code + +```elixir +User.drive({:ok, %User{}}, car_choices) +``` + +will emit a warning stating that we are passing an invalid argument: + +![Example of a warning when passing wrong argument to a function](/images/contents/type-warning-function-clause.png) + +Now consider the expression below. We are expecting the `User.drive/2` call to return `:error`, which cannot possibly be true: + +```elixir +case User.drive(user, car_choices) do + {:ok, car} -> car + :error -> Logger.error("User cannot drive") +end +``` + +Therefore the code above would emit the following warning: + +![Example of a warning when a case clause won't ever match](/images/contents/type-warning-case.png) + +Our goal is for the warnings to provide enough contextual information that lead to clear reports and that's an area we are actively looking for feedback. If you receive a warning that is unclear, please open up a bug report. + +Elixir v1.18 also augments the type system with support for tuples and lists, plus type checking of almost all Elixir language constructs, except `for`-comprehensions, `with`, and closures. Here is a non-exaustive list of the new violations that can be detected by the type system: + + * if you define a pattern that will never match any argument, such as `def function(x = y, x = :foo, y = :bar)` + + * matching or accessing tuples at an invalid index, such as `elem(two_element_tuple, 2)` + + * if you have a branch in a `try` that will never match the given expression + + * if you have a branch in a `cond` that always passes (except the last one) or always fails + + * if you attempt to use the return value of a call to `raise/2` (which by definition returns no value) + +In summary, this release takes us further in our journey of providing type checking and type inference of existing Elixir programs, without requiring Elixir developers to explicitly add type annotations. + +For existing codebases with reasonable code coverage, most type system reports will come from uncovering dead code - code which won't ever be executed - as seen in a [few](https://github.com/phoenixframework/phoenix_live_view/commit/6c6e2aaf6a01957cc6bb8a27d2513bff273e8ca2) [distinct](https://github.com/elixir-ecto/postgrex/commit/3308f277f455ec64f2d0d7be6263f77f295b1325) [projects](https://github.com/phoenixframework/flame/commit/0c0c2875e42952d2691cbdb7928fc32f4715e746). A notable example is the type system ability to track how private functions are used throughout a module and then point out which clauses are unused: + +```elixir +defmodule Example do + def public(x) do + private(Integer.parse(x)) + end + + defp private(nil), do: nil + defp private("foo"), do: "foo" + defp private({int, _rest}), do: int + defp private(:error), do: 0 + defp private("bar"), do: "bar" +end +``` + +![Example of a warning for unused private clauses](/images/contents/type-warning-private.png) + +Keep in mind the current implementation does not perform type inference of guards yet, which is an important source of typing information in programs. There is a lot the type system can learn about our codebases, that it does not yet. This brings us to the next topic. + +#### Future work + +The next Elixir release should improve the typing of maps, tuples, and closures, allowing us to type even more constructs. We also plan to fully type the `with` construct, `for`-comprehensions, as well as protocols. + +But more importantly, we want to focus on complete type inference of guards, which in turn will allow us to explore ideas such as redundant pattern matching clauses and exhaustiveness checks. Our goal with inference is to strike the right balance between developer experience, compilation times, and the ability of finding provable errors in existing codebases. You can learn more [about the trade-offs we made for inference in our documentation](https://hexdocs.pm/elixir/1.18/gradual-set-theoretic-types.html#type-inference). + +Future Elixir versions will introduce user-supplied type signatures, which should bring the benefits of a static type system without relying on inference. [Check our previous article on the overall milestones for more information](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/). + +#### Sponsors + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), [Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +## Language server listeners + +Three months ago, we welcomed [the Official Language Server team](https://elixir-lang.org/blog/2024/08/15/welcome-elixir-language-server-team/), with the goal of unifying the efforts behind code intelligence, tools, and editors in Elixir. Elixir v1.18 brings new features on this front by introducing locks and listeners to its compilation. Let's understand what it means. + +At the moment, all language server implementations have their own compilation environment. This means that your project and dependencies during development are compiled once, for your own use, and then again for the language server. This duplicate effort could cause the language server experience to lag, when it could be relying on the already compiled artifacts of your project. + +This release addresses the issue by introducing a compiler lock, ensuring that only a single operating system running Elixir compiles your project at a given moment, and by providing the ability for one operating system process to listen to the compilation results of others. In other words, different Elixir instances can now communicate over the same compilation build, instead of racing each other. + +These enhancements do not only improve editor tooling, but they also directly benefit projects like IEx and Phoenix. Here is a quick snippet showing how to enable auto-reloading inside IEx, then running `mix compile` in one shell automatically reloads the module inside the IEx session: + + + +## Built-in JSON + +[Erlang/OTP 27 added built-in support for JSON](https://www.erlang.org/doc/apps/stdlib/json.html) and we are now bringing it to Elixir. A new module, called [`JSON`](https://hexdocs.pm/elixir/1.18/JSON.html), has been added with functions to encode and decode JSON. Its most basic APIs reflect the ones [from the Jason project](https://hexdocs.pm/jason/Jason.html) (the de-facto JSON library in the Elixir community up to this point). + +A new protocol, called [`JSON.Encoder`](https://hexdocs.pm/elixir/1.18/JSON.Encoder.html), is also provided for those who want to customize how their own data types are encoded to JSON. You can also derive protocols for structs, with a single-line of code: + +```elixir +@derive {JSON.Encoder, only: [:id, :name]} +defstruct [:id, :name, :email] +``` + +The deriving API mirrors the one from `Jason`, helping those who want to migrate to the new `JSON` module. + +## Parameterized tests and ExUnit groups + +[ExUnit now supports parameterized tests](https://hexdocs.pm/ex_unit/1.18/ExUnit.Case.html#module-parameterized-tests). This allows your test modules to run multiple times under different parameters. + +For example, Elixir ships [a local, decentralized and scalable key-value process storage called `Registry`](https://hexdocs.pm/elixir/Registry.html). The registry can be partitioned and its implementation differs depending if partitioning is enabled or not. Therefore, during tests, we want to ensure both modes are exercised. With Elixir v1.18, we can achieve this by writing: + +```elixir +defmodule Registry.Test do + use ExUnit.Case, + async: true, + parameterize: [ + %{partitions: 1}, + %{partitions: 8} + ] + + # ... the actual tests ... +end +``` + +Once specified, the number of partitions is available as part of the test configuration. For example, to start one registry per test with the correct number of partitions, you can write: + +```elixir + setup config do + partitions = config.partitions + name = :"#{config.test}_#{partitions}" + opts = [keys: :unique, name: name, partitions: partitions] + start_supervised!({Registry, opts}) + opts + end +``` + +Prior to parameterized tests, Elixir resorted on code generation, which increased compilation times. Furthermore, ExUnit parameterizes the whole test modules, which also allows the different parameters to run concurrently if the `async: true` option is given. Overall, this features allows you to compile and run multiple scenarios more efficiently. + +Finally, ExUnit also comes with the ability of specifying test groups. While ExUnit supports running tests concurrently, those tests must not have shared state between them. However, in large applications, it may be common for some tests to depend on some shared state, and other tests to depend on a completely separate state. For example, part of your tests may depend on Cassandra, while others depend on Redis. Prior to Elixir v1.18, these tests could not run concurrently, but in v1.18 they might as long as they are assigned to different groups: + +```elixir +defmodule MyApp.PGTest do + use ExUnit.Case, async: true, group: :pg + + # ... +end +``` + +Tests modules within the same group do not run concurrently, but across groups, they might. + +With features like async tests, suite partitioning, and now grouping, Elixir developers have plenty of flexibility to make the most use of their machine resources, both in development and in CI. + +## `mix format --migrate` + +The `mix format` command now supports an explicit `--migrate` flag, which will convert constructs that have been deprecated in Elixir to their latest version. Because this flag rewrites the AST, it is not guaranteed the migrated format will always be valid when used in combination with macros that also perform AST rewriting. + +As of this release, the following migrations are executed: + + * Normalize parens in bitstring modifiers - it removes unnecessary parentheses in known bitstring modifiers, for example `<>` becomes `<>`, or adds parentheses for custom modifiers, where `<>` becomes `<>`. + + * Charlists as sigils - formats charlists as `~c` sigils, for example `'foo'` becomes `~c"foo"`. + + * `unless` as negated `if`s - rewrites `unless` expressions using `if` with a negated condition, for example `unless foo do` becomes `if !foo do`. We plan to deprecate `unless` in future releases. + +More migrations will be added in future releases to help us push towards more consistent codebases. + +## Summary + +Other notable changes include [`PartitionSupervisor.resize!/2`](https://hexdocs.pm/elixir/1.18/PartitionSupervisor.html#resize!/2), for resizing the number of partitions (aka processes) of a supervisor at runtime, [Registry.lock/3](https://hexdocs.pm/elixir/1.18/Registry.html#lock/3) for simple in-process key locks, PowerShell versions of `elixir` and `elixirc` scripts for better DX on Windows, and more. [See the CHANGELOG](https://hexdocs.pm/elixir/1.18/changelog.html) for the complete release notes. + +Happy coding! diff --git a/_posts/2025-01-21-remote-elixir-case.markdown b/_posts/2025-01-21-remote-elixir-case.markdown new file mode 100644 index 000000000..10d47c32d --- /dev/null +++ b/_posts/2025-01-21-remote-elixir-case.markdown @@ -0,0 +1,111 @@ +--- +layout: post +title: "Remote: growing from zero to unicorn with Elixir" +authors: +- Hugo Baraúna +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Remote. +flagship: true +logo: /images/cases/logos/remote.png +tags: growth team web +--- + +*Welcome to our series of [case studies about companies using Elixir in production](/cases.html).* + +Remote is the everywhere employment platform enabling companies to find, hire, manage, and pay people anywhere across the world. + +Founded in 2019, they reached unicorn status in just over two years and have continued their rapid growth trajectory since. + +Since day zero, Elixir has been their primary technology. Currently, their engineering organization as a whole consists of nearly 300 individuals. + +This case study focuses on their experience using Elixir in a high-growth environment. + +![Remote website screenshot](/images/cases/bg/remote.png) + +## Why Elixir? + +Marcelo Lebre, co-founder and president of Remote, had worked with many languages and frameworks throughout his career, often encountering the same trade-off: easy-to-code versus easy-to-scale. + +In 2015, while searching for alternatives, he discovered Elixir. Intrigued, Marcelo decided to give it a try and immediately saw its potential. At the time, Elixir was still in its early days, but he noticed how fast the community was growing, with support for packages and frameworks starting to show up aggressively. + +In December 2018, when Marcelo and his co-founder decided to start the company, they had to make a decision about the technology that would support their vision. Marcelo wanted to prioritize building a great product quickly without worrying about scalability issues from the start. He found Elixir to be the perfect match: + +> I wanted to focus on building a great product fast and not really worry about its scalability. Elixir was the perfect match—reliable performance, easy-to-read syntax, strong community, and a learning curve that made it accessible to new hires. +> +> \- Marcelo Lebre, Co-founder and President + +The biggest trade-off Marcelo identified was the smaller pool of Elixir developers compared to languages like Ruby or Python. However, he quickly realized that the quality of candidates more than made up for it: + +> The signal-to-noise ratio in the quality of Elixir candidates was much higher, which made the trade-off worthwhile. +> +> \- Marcelo Lebre, Co-founder and President + +## Growing with a monolithic architecture + +Remote operates primarily with a monolith, with Elixir in the backend and React in the front-end. + +The monolith enabled speed and simplicity, allowing the team to iterate quickly and focus on building features. However, as the company grew, they needed to invest in tools and practices to manage almost 180 engineers working in the same codebase. + +One practice was boundary enforcement. They used the [Boundary library](https://github.com/sasa1977/boundary) to maintain strict boundaries between modules and domains inside the codebase. + +Another key investment was optimizing their compilation time in the CI pipeline. Since their project has around 15,000 files, compiling it in every build would take too long. So, they implemented incremental builds in their CI pipeline, recompiling only the files affected by changes instead of the entire codebase. + +> I feel confident making significant changes in the codebase. The combination of using a functional language and our robust test suite allows us to keep moving forward without too much worry. +> +> \- André Albuquerque, Staff Engineer + +Additionally, as their codebase grew, the Elixir language continued to evolve, introducing better abstractions for developers working with large codebases. For example, with the release of Elixir v1.11, the [introduction of config/runtime.exs](/blog/2020/10/06/elixir-v1-11-0-released/) provided the Remote team with a better foundation for managing configuration. This enabled them to move many configurations from compile-time to runtime, significantly reducing unnecessary recompilations caused by configuration updates. + +## Infra-structure and operations + +One might expect Remote’s infrastructure to be highly complex, given their global scale and the size of their engineering team. Surprisingly, their setup remains relatively simple, reflecting a thoughtful balance between scalability and operational efficiency. + +Remote runs on AWS, using EKS (Elastic Kubernetes Service). The main backend (the monolith) operates in only five pods, each with 10 GB of memory. They use [Distributed Erlang](https://www.erlang.org/doc/system/distributed.html) to connect the nodes in their cluster, enabling seamless communication between processes running on different pods. + +For job processing, they rely on [Oban](https://github.com/oban-bg/oban), which runs alongside the monolith in the same pods. + +Remote also offers a public API for partners. While this API server runs separately from the monolith, it is the same application, configured to start a different part of its supervision tree. The separation was deliberate, as the team anticipated different load patterns for the API and wanted the flexibility to scale it independently. + +The database setup includes a primary PostgreSQL instance on AWS RDS, complemented by a read-replica for enhanced performance and scalability. Additionally, a separate Aurora PostgreSQL instance is dedicated to storing Oban jobs. Over time, the team has leveraged tools like PG Analyze to optimize performance, addressing bottlenecks such as long queries and missing indexes. + +This streamlined setup has proven resilient, even during unexpected spikes in workload. The team shared an episode where a worker’s job count unexpectedly grew by two orders of magnitude. Remarkably, the system handled the increase seamlessly, continuing to run as usual without requiring any design changes or manual intervention. + +> We once noticed two weeks later that a worker’s load had skyrocketed. But the scheduler worked fine, and everything kept running smoothly. That was fun. +> +> \- Alex Naser, Staff Engineer + +## Team organization and responsibilities + +Around 90% of their backend team works in the monolith, while the rest work in a few satellite services, also written in Elixir. + +Within the monolith, teams are organized around domains such as onboarding, payroll, and billing. Each team owns one or multiple domains. + +To streamline accountability in a huge monolith architecture, Remote invested heavily in team assignment mechanisms. + +They implemented a tagging system that assigns ownership down to the function level. This means any trace—whether sent to tools like Sentry or Datadog—carries a tag identifying the responsible team. This tagging also extends to endpoints, allowing teams to monitor their areas effectively and even set up dashboards for alerts, such as query times specific to their domain. + +The tagging system also simplifies CI workflows. When a test breaks, it’s automatically linked to the responsible team based on the Git commit. This ensures fast issue identification and resolution, removing the need for manual triaging. + +## Hiring and training + +Remote’s hiring approach prioritizes senior engineers, regardless of their experience with Elixir. + +During the hiring process, all candidates are required to complete a coding exercise in Elixir. For those unfamiliar with the language, a tailored version of the exercise is provided, designed to introduce them to Elixir while reflecting the challenges they would face if hired. + +Once hired, new engineers are assigned an engineering buddy to guide them through the onboarding process. + +For hires without prior Elixir experience, Remote developed an internal Elixir training camp, a curated collection of best practices, tutorials, and other resources to introduce new hires to the language and ecosystem. This training typically spans two to four weeks. + +After completing the training, engineers are assigned their first tasks—carefully selected tickets designed to build confidence and familiarity with the codebase. + +## Summing up + +Remote’s journey highlights how thoughtful technology, infrastructure, and team organization decisions can support rapid growth. + +By leveraging Elixir’s strengths, they built a monolithic architecture that balanced simplicity with scalability. This approach allowed their engineers to iterate quickly in the early stages while effectively managing the complexities of a growing codebase. + +Investments in tools like the Boundary library and incremental builds ensured their monolith remained efficient and maintainable even as the team and codebase scaled dramatically. + +Remote's relatively simple infrastructure demonstrates that scaling doesn't always require complexity. Their ability to easily handle unexpected workload spikes reflects the robustness of their architecture and operational practices. + +Finally, their focus on team accountability and streamlined onboarding allowed them to maintain high productivity while integrating engineers from diverse technical backgrounds, regardless of their prior experience with Elixir. diff --git a/_posts/2025-02-26-elixir-openchain-certification.markdown b/_posts/2025-02-26-elixir-openchain-certification.markdown new file mode 100644 index 000000000..76285ea70 --- /dev/null +++ b/_posts/2025-02-26-elixir-openchain-certification.markdown @@ -0,0 +1,72 @@ +--- +layout: post +title: "Announcing Elixir OpenChain Certification" +authors: + - Jonatan Männchen + - José Valim +category: Announcements +excerpt: "The Elixir project now meets OpenChain (ISO/IEC 5230). Each release ships with Source SBoMs in CycloneDX 1.6 and SPDX 2.3, plus attestation." +tags: openchain compliance +--- + +We are pleased to share that the Elixir project now complies with +[OpenChain][openchain] ([ISO/IEC 5230][iso_5230]), an international +standard for open source license compliance. This step aligns with broader +efforts to meet industry standards for supply chain and cybersecurity best +practices. + +“Today’s announcement around Elixir’s conformance represents another significant +example of community maturity,” says Shane Coughlan, OpenChain General Manager. +“With projects - the final upstream - using ISO standards for compliance and +security with increasing frequency, we are seeing a shift to longer-term +improvements to trust in the supply chain.” + +## Why OpenChain Compliance Helps + +By following OpenChain (ISO/IEC 5230), we demonstrate clear processes around +license compliance. This benefits commercial and community users alike, making +Elixir easier to adopt and integrate with confidence. + +## Changes for Elixir Users + +Elixir has an automated release process where its artifacts are signed. This +change strengthens this process by: + +- All future Elixir releases will include a Source SBoM in + [CycloneDX 1.6 or later][cyclonedx] and [SPDX 2.3 or later][spdx] formats. +- Each release will be attested along with the Source SBoM. + +These additions offer greater transparency into the components and licenses of +each release, supporting more rigorous supply chain requirements. + +## Changes for Contributors + +Contributing to Elixir remains largely the same, we have added more clarity and +guidelines around it: + +- Contributions remain under the Apache-2.0 License. Other licenses cannot be + accepted. +- The project now enforces the [Developer Certificate of Origin (DCO)][dco], + ensuring clarity around contribution ownership. + +Contributors will notice minimal procedural changes, as standard practices +around licensing remain in place. + +For more details, see the [CONTRIBUTING guidelines][contributing]. + +## Commitment + +These updates were made in collaboration with the +[Erlang Ecosystem Foundation][erlef], reflecting a shared +commitment to robust compliance and secure development practices. Thank you to +everyone who supported this milestone. We appreciate the community’s ongoing +contributions and look forward to continuing the growth of Elixir under these +established guidelines. + +[openchain]: https://openchainproject.org/ +[erlef]: https://erlef.org/ +[spdx]: https://spdx.org/rdf/terms/ +[cyclonedx]: https://cyclonedx.org/specification/overview/ +[iso_5230]: https://www.iso.org/standard/81039.html +[dco]: https://developercertificate.org/ +[contributing]: https://github.com/elixir-lang/elixir/blob/main/CONTRIBUTING.md diff --git a/_posts/2025-03-25-cyanview-elixir-case.markdown b/_posts/2025-03-25-cyanview-elixir-case.markdown new file mode 100644 index 000000000..48c021eae --- /dev/null +++ b/_posts/2025-03-25-cyanview-elixir-case.markdown @@ -0,0 +1,92 @@ +--- +layout: post +title: "Cyanview: Coordinating Super Bowl's visual fidelity with Elixir" +authors: +- Lars Wikman +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Cyanview. +logo: /images/cases/logos/cyanview.png +tags: superbowl mqtt +--- + +How do you coordinate visual fidelity across two hundred cameras for a live event like the Super Bowl? + +The answer is: by using the craft of camera shading, which involves adjusting each camera to ensure they match up in color, exposure and various other visual aspects. The goal is to turn the live event broadcast into a cohesive and consistent experience. For every angle used, you want the same green grass and the same skin tones. Everything needs to be very closely tuned across a diverse set of products and brands. From large broadcast cameras, drone cameras, and PTZ cameras to gimbal-held mirrorless cameras and more. This is what Cyanview does. Cyanview is a small Belgian company that sells products for the live video broadcast industry, and its primary focus is shading. + +Broadcast is a business where you only get one chance to prove that your tool is up to the task. Reliability is king. There can be no hard failures. + +A small team of three built a product so powerful and effective that it spread across the industry purely on the strength of its functionality. Without any marketing, it earned a reputation among seasoned professionals and became a staple at the world’s top live events. Cyanview's Remote Control Panel (RCP) is now used by specialist video operators on the Olympics, Super Bowl, NFL, NBA, ESPN, Amazon and many more. Even most fashion shows in Paris use Cyanview’s devices. + +These devices put Elixir right in the critical path for serious broadcast operations. By choosing Elixir, Cyanview gained best-in-class networking features, state-of-the-art resilience and an ecosystem that allowed fast iteration on product features. + +![Operating many displays with Cyanview products](/images/cases/bg/cyanview-4.jpg "Operating many displays with Cyanview products") + +## Why Elixir? + +The founding team of Cyanview primarily had experience with embedded development, and the devices they produce involve a lot of low-level C code and plenty of FPGA. This is due to the low-level details of color science and the really tight timing requirements. + +If you’ve ever worked with camera software, you know it can be a mixed bag. Even after going fully digital, much of it remained tied to analog systems or relied on proprietary connectivity solutions. Cyanview has been targeting IP (as in Internet Protocol) from early on. This means Cyanview's software can operate on commodity networks that work in well-known and well-understood ways. This has aligned well with an increase in remote production, partially due to the pandemic, where production crews operate from a central location with minimal crew on location. Custom radio frequency or serial wire protocols have a hard time scaling to cross-continent distances. + +This also paved the way for Elixir, as the Erlang VM was designed to communicate and coordinate millions of devices, reliably, over the network. + +Elixir was brought in by the developer Ghislain, who needed to build integrations with cameras and interact with the other bits of required video gear, with many different protocols over the network. The language comes with a lot of practical features for encoding and decoding binary data down to the individual bits. Elixir gave them a strong foundation and the tools to iterate fast. + +Ghislain has been building the core intellectual property of Cyanview ever since. While the physical device naturally has to be solid, reliable, and of high quality, a lot of the secret sauce ultimately lies in the massive number of integrations and huge amounts of reverse engineering. Thus, the product is able to work with as many professional camera systems and related equipment as possible. It is designed to be compatible with everything and anything a customer is using. Plus, it offers an API to ensure smooth integration with other devices. + +David Bourgeois, the founder of Cyanview, told us a story how these technical decisions alongside Elixir helped them tackle real-world challenges: + +"During the Olympics in China, a studio in Beijing relied on a large number of Panasonic PTZ cameras. Most of their team, however, was based in Paris and needed to control the cameras remotely to run various shows throughout the day. The problem? Panasonic’s camera protocols were never designed for internet use — they require precise timing and multiple messages for every adjustment. With network latency, that leads to timeouts, disconnects, and system failures... So they ended up placing our devices next to the cameras in Beijing and controlled them over IP from Paris — just as designed." + +![Cyanview RIO device mounted on a camera at a sports field](/images/cases/bg/cyanview-2.jpg "Cyanview RIO device mounted on a camera at a sports field") + +The devices in a given location communicate and coordinate on the network over a custom MQTT protocol. Over a hundred cameras without issue on a single Remote Control Panel (RCP), implemented on top of Elixir's network stack. + +## Technical composition + +The system as a whole consists of RCP devices running a Yocto Linux system, with most of the logic built in Elixir and C. While Python is still used for scripting and tooling, its role has gradually diminished. The setup also includes multiple microcontrollers and the on-camera device, all communicating over MQTT. Additionally, cloud relays facilitate connectivity, while dashboards and controller UIs provide oversight and control. The two critical devices are the RCP offering control on the production end and the RIO handling low-latency manipulation of the camera. Both run Elixir. + +The configuration UI is currently built in Elm, but - depending on priorities - it might be converted to [Phoenix LiveView](https://phoenixframework.org/) over time to reduce the number of languages in use. The controller web UI is already in LiveView, and it is performing quite well on a very low-spec embedded Linux machine. + +The cloud part of the system is very limited today, which is unusual in a world of SaaS. There are cloud relays for distributing and sharing camera control as well as forwarding network ports between locations and some related features, also built in Elixir, but cloud is not at the core of the business. The devices running Elixir on location form a cluster over IP using a custom MQTT-based protocol suited to the task and are talking to hundreds of cameras and other video devices. + +It goes without saying that integration with so much proprietary equipment comes with challenges. Some integrations are more reliable than others. Some devices are more common, and their quirks are well-known through hard-won experience. A few even have good documentation that you can reference while others offer mystery and constant surprises. In this context, David emphasizes the importance of Elixir's mechanisms for recovering from failures: + +"If one camera connection has a blip, a buggy protocol or the physical connection to a device breaks it is incredibly important that everything else keeps working. And this is where Elixir’s supervision trees provide a critical advantage." + +## Growth & team composition + +The team has grown over the 9 years that the company has been operating, but it did so at a slow and steady pace. On average, the company has added just one person per year. With nine employees at the time of writing, Cyanview supports some of the biggest broadcast events in the world. + +There are two Elixir developers on board: Daniil who is focusing on revising some of the UI as well as charting a course into more cloud functionality, and Ghislain, who works on cameras and integration. Both LiveView and Elm are used to power device UIs and dashboards. + +What’s interesting is that, overall, the other embedded developers say that they don't know much about Elixir and they don't use it in their day-to-day work. Nonetheless, they are very comfortable implementing protocols and encodings in Elixir. The main reason they haven’t fully learned the language is simply time — they have plenty of other work to focus on, and deep Elixir expertise hasn’t been necessary. After all, there’s much more to their work beyond Elixir: designing PCBs, selecting electronic components, reverse engineering protocols, interfacing with displays, implementing FPGAs, managing production tests, real productions and releasing firmware updates. + +## Innovation and customer focus + +![Operator using Cyanview RCP for a massive crowd in an arena](/images/cases/bg/cyanview-3.jpg "Operator using Cyanview RCP for a massive crowd in an arena") + +Whether it’s providing onboard cameras in 40+ cars during the 24 hours of Le Mans, covering Ninja Warrior, the Australian Open, and the US Open, operating a studio in the Louvre, being installed in NFL pylons, or connecting over 200 cameras simultaneously – the product speaks for itself. Cyanview built a device for a world that runs on top of IP, using Elixir, a language with networking and protocols deep in its bones. This choice enabled them to do both: implement support for all the equipment and provide features no one else had. + +By shifting from conventional local-area radio frequency, serial connections, and inflexible proprietary protocols to IP networking, Cyanview’s devices redefined how camera systems operate. Their feature set is unheard of in the industry: Unlimited multicam. Tally lights. Pan & Tilt control. Integration with color correctors. World-spanning remote production. + +The ease and safety of shipping new functionality have allowed the company to support new features very quickly. One example is the increasing use of mirrorless cameras on gimbals to capture crowd shots. Cyanview were able to prototype gimbal control, test it with a customer and validate that it worked in a very short amount of time. This quick prototyping and validation of features is made possible by a flexible architecture that ensures that critical fundamentals don't break. + +Camera companies that don't produce broadcast shading remotes, such as Canon or RED, recommend Cyanview to their customers. Rather than competing with most broadcast hardware companies, Cyanview considers itself a partner. The power of a small team, a quality product and powerful tools can be surprising. Rather than focusing on marketing, Cyanview works very closely with its customers by supporting the success of their events and providing in-depth customer service. + +## Looking back and forward + +When asked if he would choose Elixir again, David responded: + +"Yes. We've seen what the Erlang VM can do, and it has been very well-suited to our needs. You don't appreciate all the things Elixir offers out of the box until you have to try to implement them yourself. It was not pure luck that we picked it, but we were still lucky. Elixir turned out to bring a lot that we did not know would be valuable to us. And we see those parts clearly now." + +Cyanview hopes to grow the team more, but plans to do so responsibly over time. Currently there is a lot more to do than the small team can manage. + +Development is highly active, with complementary products already in place alongside the main RCP device, and the future holds even more in that regard. Cloud offerings are on the horizon, along with exciting hardware projects that build on the lessons learned so far. As these developments unfold, we’ll see Elixir play an increasingly critical role in some of the world’s largest live broadcasts. + +![Cyanview Remote Control Panels in a control room](/images/cases/bg/cyanview-1.jpg "Cyanview Remote Control Panels in a control room") + +## In summary + +A high-quality product delivering the right innovation at the right time in an industry that's been underserved in terms of good integration. Elixir provided serious leverage for developing a lot of integrations with high confidence and consistent reliability. In an era where productivity and lean, efficient teams are everything, Cyanview is a prime example of how Elixir empowers small teams to achieve an outsized impact. + diff --git a/_posts/2025-06-02-elixir-outreach-stipend-for-speakers.markdown b/_posts/2025-06-02-elixir-outreach-stipend-for-speakers.markdown new file mode 100644 index 000000000..1bca122fc --- /dev/null +++ b/_posts/2025-06-02-elixir-outreach-stipend-for-speakers.markdown @@ -0,0 +1,42 @@ +--- +layout: post +title: "Elixir Outreach stipend for speakers and trainers" +authors: +- José Valim +category: Announcements +excerpt: "We announce the Elixir Outreach stipend in partnership with Dashbit, Oban, and the Erlang Ecosystem Foundation" +--- + +[Dashbit](https://dashbit.co), [Oban](https://oban.pro), and the [Erlang Ecosystem Foundation (EEF)](https://erlef.org) are glad to announce a new program, which we will trial over the next 12 months, called "Elixir Outreach". Our goal is to provide funds to community members who want to present Elixir and Erlang to other ecosystems and communities, while respecting our joint values. + +In a nutshell: + +* We will provide funds to community members to speak **in-person** about anything related to Elixir and the Erlang VM. + +* We will cover hotel and transportation costs for up to $700 USD. Please reach out, even if you expect to exceed that limit. This is our first time running the program and we're refining the budget. + +* The event must expect at least 150 attendees and happen outside of the Elixir, overall BEAM, and functional programming communities. In other words, we won't cover costs for attending Erlang, Elixir, or other BEAM/FP conferences nor meetups. Consider it as an opportunity to learn and bring external knowledge and experiences to the BEAM community. + +* You will be expected to send a report about your experience. The format and duration is up to you. We'd prefer that you write a blog post or an article sharing your overall experience with the involved communities. However, if you would prefer to only send it privately to us, that's fine too! + +The event should take place within your area. Our overall goal is to support multiple individuals, rather than drain our budget on a few long-distance flights (such as across coasts or continents). We are flexible on event location, distance, or type. If in doubt, reach out to [elixir_outreach at erlef dot org](mailto:elixir_outreach@erlef.org) + +Our initial budget of $7000 was donated by Dashbit ($5000) and Oban ($2000) to the Erlang Ecosystem Foundation (EEF), specifically for this program. The EEF will oversee the distribution of the funds. + +## Requesting a stipend + +To request a stipend, visit the [Erlang Ecosystem Foundation website and choose "Elixir Outreach" as the stipend type](https://erlef.org/stipends/form?type=elixir-outreach). + +Given we have limited funds, we cannot guarantee they will be available when you request them. We recommend reaching out to us before submitting or acceptance your talk. Therefore, by contacting us early, we can validate if the event matches the criteria above, ask questions, and earmark the funds. Once your talk is accepted, send us any itemized travel and accommodation costs so we can transfer the stipend to you, (not in excess of $700 USD). + +You can also request a stipend after your talk has already been accepted, but then there are no guarantees a stipend will be available. + +Our goal is to make this process simple and as straight-forward as possible. Although, we reserve the right to refuse a request for any reason. If in doubt, reach out to [elixir_outreach at erlef dot org](mailto:elixir_outreach@erlef.org). + +## Acknowledgements + +This is a new effort for all involved! Please be patient while we figure out the details. + +If you are looking for conferences to speak at, [Dave Aronson keeps a list of CFPs closing soon](https://www.codosaur.us/speaking/cfps-ending-soon) and there are likely others available. Note, we don’t necessarily endorse all of the conferences listed nor guarantee they meet the requirements above, but the list may help you get the ball rolling. + +Thanks to Parker Selbert, Shannon Selbert, Brian Cardarella, Alistair Woodman, and Lee Barney for feedback and helping make this a reality. \ No newline at end of file diff --git a/_posts/2025-08-05-global-elixir-meetups.markdown b/_posts/2025-08-05-global-elixir-meetups.markdown new file mode 100644 index 000000000..df098c1d6 --- /dev/null +++ b/_posts/2025-08-05-global-elixir-meetups.markdown @@ -0,0 +1,20 @@ +--- +layout: post +title: "Take part in the Global Elixir Meetups week" +authors: +- José Valim +category: Announcements +excerpt: "We are launching Global Elixir Meetups - a week where the Elixir community organizes meetups around the world to meet, learn from each other, and discuss everything related to Elixir and the Erlang VM." +--- + +> Update: Our first Global Elixir Meetups was a success with 46 meetups spread across six continents. Thanks to everyone who organized and attended. See you next time! + +We are launching [Global Elixir Meetups (GEMs)](https://globalelixirmeetups.com) - a week where the Elixir community organizes meetups around the world to meet and learn from each other. Our goal is to spark local communities, old and new, to get together and discuss everything related to Elixir and the Erlang VM. + +Our first GEM will happen on 22-28 September and, if you were looking for an opportunity to organize a meetup, now is the perfect time: visit [the Global Elixir Meetups website](https://globalelixirmeetups.com) to learn how to organize and attend. Organizers may also opt-in to live stream their meetups to the whole world directly from the website. + +![Global Elixir Meetup banner](/images/contents/gem.jpeg "Global Elixir Meetup banner") + +The Global Elixir Meetup is organized by [Software Mansion](https://swmansion.com), who brought their expertise as creators of [Membrane](https://membrane.stream) and [Elixir WebRTC](https://elixir-webrtc.org) to make it all possible. At launch, we are already counting with 7 meetups across Europe, South America, and North America, with hopefully more continents joining us soon. + +Go ahead and find your closest GEM or run your own! diff --git a/_posts/2025-08-18-interop-and-portability.markdown b/_posts/2025-08-18-interop-and-portability.markdown new file mode 100644 index 000000000..dc4895db8 --- /dev/null +++ b/_posts/2025-08-18-interop-and-portability.markdown @@ -0,0 +1,279 @@ +--- +layout: post +title: "Interoperability in 2025: beyond the Erlang VM" +authors: +- Wojtek Mach +- José Valim +category: Announcements +excerpt: "We explore the mechanisms for interoperability and portability between Elixir, other programming languages, and runtimes." +--- + +The Erlang Virtual Machine has, historically, provided three main options for interoperability with other languages and ecosystems, with different degrees of isolation: + +* [NIFs (Native Implemented Functions)](https://www.erlang.org/doc/apps/erts/erl_nif.html) integrate with third party code in the same memory space via C bindings. This translates to low overhead and best performance but it also means faulty code can bring the whole Virtual Machine down, bypassing some of Erlang's fault-tolerance guarantees + +* [Ports](https://www.erlang.org/doc/system/ports.html) start a separate Operating System process to communicate with other languages through STDIN/STDOUT, guaranteeing process isolation. In a typical Erlang fashion, ports are fully evented, concurrent, and distributed (i.e. you can pass and communicate with ports across nodes) + +* [Distributed nodes](https://www.erlang.org/doc/apps/erts/erl_dist_protocol.html) rely on Erlang well-defined distribution and serialization protocol to communicate with other runtimes. Any language can implement said protocol and act as an Erlang node, giving you full node isolation between runtimes + +Those mechanisms have led to multiple integrations between Elixir and other programming languages, such as Zig and Rust, and more recently C++, Python, and Swift, which we will explore here. + +Furthermore, alternative implementations of the Erlang VM and Elixir have brought a fourth category of **interoperability through portability**: where your Elixir program runs in a completely different environment to leverage its native capabilities, libraries, and ecosystem, while maintaining Elixir's syntax and semantics (either partially or fully). This opens up some exciting new possibilities and since this approach is still relatively uncharted territory, let's dive into it first. + +## Portability + +The [AtomVM](https://atomvm.org) is a lightweight implementation of the Erlang VM that can run on constrained environments, such as microcontrollers with just a few hundred kilobytes of memory such as ESP32, STM32 or Pico. AtomVM supports a functional subset of Erlang VM and its standard library, all optimized to run on tiny microcontrollers. + +Given its low footprint, AtomVM can also target WebAssembly, paving the way to run Elixir in web browsers and alternative WASM runtimes in the future. The [Popcorn](https://popcorn.swmansion.com) project, [recently announced at ElixirConf EU 2025](https://www.youtube.com/watch?v=ep--rQO1FRI), builds on those capabilities to provide better interoperability between Elixir and JavaScript. + +### Popcorn + +[Popcorn](https://popcorn.swmansion.com) is a library for running Elixir in web browsers, with JavaScript interoperability. Popcorn brings an extensive subset of Elixir semantics into the browser and, although it is in its infancy, it is already capable of [running interactive Elixir code entirely client side](https://popcorn.swmansion.com/demos/eval). + +And here is a quick example showing how to communicate with JavaScript from WASM: + +```elixir +defmodule HelloPopcorn do + use GenServer + + @process_name :main + + def start_link(args) do + GenServer.start_link(__MODULE__, args, name: @process_name) + end + + @impl true + def init(_init_arg) do + Popcorn.Wasm.register(@process_name) + IO.puts("Hello console!") + + Popcorn.Wasm.run_js(""" + () => { + document.body.innerHTML = "Hello from WASM!"; + } + """) + + :ignore + end +end +``` + +Popcorn could help with Elixir adoption by making it really easy to create interactive guides with executable code right there in the browser. And once it's production ready, it could enable offline, local-first applications, entirely in Elixir. + +### Hologram + +[Hologram](https://hologram.page) is a full-stack isomorphic Elixir web framework that runs on top of Phoenix. It lets developers create dynamic, interactive web applications entirely in Elixir. + +Hologram transpiles Elixir code to JavaScript and provides a complete framework including templates, components, routing, and client-server communication for building rich web applications. + +Here is a snippet of a Hologram component that handles drawing events entirely client-side, taken from the official [SVG Drawing Demo](https://hologram.page/demos/svg-drawing): + +```elixir +defmodule DrawingBoard do + use Hologram.Component + + def init(_props, component, _server) do + put_state(component, drawing?: false, path: "") + end + + def template do + ~HOLO""" + + + + """ + end + + def action(:draw_move, params, component) when component.state.drawing? do + new_path = component.state.path <> " L #{params.event.offset_x} #{params.event.offset_y}" + put_state(component, :path, new_path) + end + + def action(:start_drawing, params, component) do + new_path = component.state.path <> " M #{params.event.offset_x} #{params.event.offset_y}" + put_state(component, drawing?: true, path: new_path) + end +end +``` + +While Popcorn runs on a lightweight implementation of the Erlang VM with all of its primitives, Hologram works directly on the Elixir syntax tree. They explore distinct paths for bringing Elixir to the browser and are both in active development. + +## Native Implemented Functions (NIFs) + +NIFs allow us to write performance-critical or system-level code and call it directly from Erlang and Elixir as if it were a regular function. + +NIFs solve practical problems like improving performance or using all Operating System capabilities. NIFs run in the same Operating System process as the VM, the same memory space. With them we can use third-party native libraries, execute syscalls, interface with the hardware, etc. On the other hand, using them can forgo some of Erlang's stability and error handling guarantees. + +Originally, NIFs could never block and had to be written in a "yielding" fashion, which limited their applicability. Since Erlang/OTP 17, however, NIFs can be scheduled to run on separate OS threads called "dirty schedulers", based on their workloads (IO or CPU). This has directly brought Elixir and the Erlang VM into new domains, such as [Numerical Elixir](https://github.com/elixir-nx), and to interop with new languages and ecosystems. + +### C + +Erlang's NIFs directly target the C programming language and is used to implement low-level functionality present in Erlang's standard library: + +```c +#include + +static ERL_NIF_TERM add_int64_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int64_t a, b; + if (!enif_get_int64(env, argv[0], &a) || !enif_get_int64(env, argv[1], &b)) { + return enif_make_badarg(env); + } + return enif_make_int64(env, a + b); +} + +static ErlNifFunc nif_funcs[] = { + {"add", 2, add_int64_nif}, +}; + +ERL_NIF_INIT("Elixir.Example", nif_funcs, NULL, NULL, NULL, NULL) +``` + +Writing NIFs in C can be verbose and error-prone. Fortunately, the Elixir ecosystem offers a number of high-quality libraries that make it possible to write NIFs in other languages, let's check them out. + +### C++ + +[Fine](https://github.com/elixir-nx/fine) is a lightweight C++ library that wraps the NIF API with a modern interface. Given the widespread use of C++ in machine learning and data, Fine aims to reduce the friction of getting from Elixir to C++ and vice-versa. + +Here's the same NIF that adds two numbers in C++, using Fine: + +```c++ +#include + +int64_t add(ErlNifEnv *env, int64_t a, int64_t b) { + return a + b; +} + +FINE_NIF(add, 0); +FINE_INIT("Elixir.Example"); +``` + +Fine automatically encodes and decodes NIF arguments and return values based on the function signature, significantly reducing boilerplate code. It also has first-class support for Elixir structs, propagating C++ exceptions as Elixir exceptions, and more. + +### Rust + +[Rustler](https://github.com/rusterlium/rustler) is a library for writing NIFs in Rust. The goal is to make it impossible to crash the VM when using "safe" Rust code. Furthermore, Rustler makes it easy to encode/decode Rust values to and from Elixir terms while safely and ergonomically managing resources. + +Here's an example NIF implemented with Rustler: + +```rust +#[rustler::nif] +fn add(a: i64, b: i64) -> i64 { + a + b +} + +rustler::init!("Elixir.Example"); +``` + +### Zig + +[Zigler](https://hexdocs.pm/zigler) lets us write NIFs in Zig, a low-level programming language designed for maintaining robust, optimal, and reusable software. Zig removes hidden control flow, implicit memory allocation, and similar abstractions in favour of code that's explicit and predictable. + +Zigler compiles Zig code at build time and exposes it directly to Elixir, without external build scripts or glue. It tightly integrates with Elixir tooling: Zig code is formatted via `mix format` and documentation written in Zig appears in IEx via the `h` helper. + +Here's an example NIF in Zig: + +```elixir +iex> Mix.install([:zigler]) +iex> defmodule Example do + use Zig, otp_app: :zigler + + ~Z""" + pub fn add(a: i64, b: i64) i64 { + return a + b; + } + """ + end +iex> Example.add(1, 2) +3 +``` + +We can write NIFs directly in IEx sessions, scripts, Livebook notebooks, and similar! And with Zig's excellent interop with C, it's really easy to experiment with native code on the Erlang VM. + +### Python + +[Pythonx](https://github.com/livebook-dev/pythonx) runs a Python interpreter in the same OS process as your Elixir application, allowing you to evaluate Python code and conveniently convert between Python and Elixir data structures. Pythonx also integrates with the [uv](https://docs.astral.sh/uv/) package manager, automating the management of Python and its dependencies. + +One caveat is that Python's Global Interpreter Lock (GIL) prevents multiple threads from executing Python code at the same time so calling Pythonx from multiple Elixir processes does not provide concurrency we might expect and can become source of bottlenecks. However, GIL is a constraint for regular Python code only. Packages with CPU-intense functionality, such as `numpy`, have native implementation of many functions and invoking those releases the GIL (GIL is also released when waiting on I/O). + +Here's an example of using `numpy` in Elixir: + +```elixir +iex> Mix.install([{:pythonx, "~> 0.4.0"}]) +iex> Pythonx.uv_init(""" + [project] + name = "myapp" + version = "0.0.0" + requires-python = "==3.13.*" + dependencies = [ + "numpy==2.2.2" + ] + """) +iex> import Pythonx, only: :sigils +iex> x = 1 +iex> ~PY""" + import numpy as np + + a = np.int64(x) + b = np.int64(2) + a + b + """ +#Pythonx.Object< + np.int64(3) +> +``` + +[Livebook](https://livebook.dev) uses Pythonx to allow Elixir and Python code cells to co-exist in the same notebook (and in the same memory space), with low-overhead when transferring data between them. + +## Distributed nodes + +Elixir, by way of Erlang, has built-in support for distributed systems. Multiple nodes can connect over a network and communicate using message passing, with the same primitives such as `send` and `receive` used for both local and remote processes. + +Nodes become discoverable in the cluster simply by starting them with names. Once we connect to a node, we can send messages, spawn remote processes, and more. Here's an example: + +```shell +$ iex --name a@127.0.0.1 --cookie secret +$ iex --name b@127.0.0.1 --cookie secret +iex(a@127.0.0.1)> Node.connect(:"b@127.0.0.1") +iex(a@127.0.0.1)> node() +:"a@127.0.0.1" +iex(a@127.0.0.1)> :erpc.call(:"b@127.0.0.1", fn -> node() end) +:"b@127.0.0.1" +``` + +While Distributed Erlang is typically used for Erlang-Erlang communication, it can be also used for interacting with programs written in other programming languages. Erlang/OTP includes [Erl_Interface](https://www.erlang.org/doc/apps/erl_interface), a C library for writing programs that can participate in the Erlang cluster. Such programs are commonly called C nodes. + +Any language may implement these protocols from scratch or, alternatively, use `erl_interface` as its building block. For example, Erlang/OTP ships with [Jinterface](https://www.erlang.org/doc/apps/jinterface) application, a Java library that lets JVM programs act as distributed Erlang nodes. Another recent example is the [Swift Erlang Actor System](https://github.com/otp-interop/swift-erlang-actor-system), for communicating between Swift and Erlang/Elixir programs. + +## Ports + +Last but not least, ports are the basic mechanism that Elixir/Erlang uses to communicate with the outside world. Ports are the most common of interoperability across programming languages, so we will only provide two brief examples. + +In Elixir, the `Port` module offers a low-level API to start separate programs. Here's an example that runs `uname -s` to print the current operating system: + +```elixir +iex> port = Port.open({:spawn, "uname -s"}, [:binary]) +iex> flush() +{#Port<0.3>, {:data, "Darwin\n"}} +iex> send(port, {self(), :close}) +iex> flush() +{#Port<0.3>, :closed} +:ok +``` + +Most times, however, developers use `System.cmd/3` to invoke short-running programs: + +```elixir +iex> System.cmd("uname", ["-s"]) +{"Darwin\n", 0} +``` + +## Summary + +This article highlights many of the several options for interoperating with Elixir and the Erlang Virtual Machine. While it does not aim to be a complete reference, it covers integration across a range of languages, such as Rust, Zig, Python and Swift, as well as portability to different environments, including microcontrollers and web browsers. diff --git a/_posts/2025-10-16-elixir-v1-19-0-released.markdown b/_posts/2025-10-16-elixir-v1-19-0-released.markdown new file mode 100644 index 000000000..32c2ee283 --- /dev/null +++ b/_posts/2025-10-16-elixir-v1-19-0-released.markdown @@ -0,0 +1,219 @@ +--- +layout: post +title: "Elixir v1.19 released: enhanced type checking and up to 4x faster compilation for large projects" +authors: +- José Valim +category: Releases +excerpt: "Elixir v1.19 released: type checking of protocols and anonymous functions, broader type inference, improved compile times, and more" +--- + +Elixir v1.19 brings further improvements to the type system and compilation times, allowing us to find more bugs, faster. + +## Type system improvements + +This release improves the type system by adding type inference of anonymous functions and type checking of protocols. These enhancements seem simple on the surface but required us to go beyond existing literature by extending current theory and developing new techniques. We will outline the technical details in future articles. For now, let's look at what's new. + +### Type checking of protocol dispatch and implementations + +This release adds type checking when dispatching and implementing protocols. + +For example, string interpolation in Elixir uses the `String.Chars` protocol. If you pass a value that does not implement said protocol, Elixir will now emit a warning accordingly. + +Here is an example passing a range, which cannot be converted into a string, to an interpolation: + +```elixir +defmodule Example do + def my_code(first..last//step = range) do + "hello #{range}" + end +end +``` + +the above emits the following warnings: + +``` +warning: incompatible value given to string interpolation: + + data + +it has type: + + %Range{first: term(), last: term(), step: term()} + +but expected a type that implements the String.Chars protocol, it must be one of: + + dynamic( + %Date{} or %DateTime{} or %NaiveDateTime{} or %Time{} or %URI{} or %Version{} or + %Version.Requirement{} + ) or atom() or binary() or float() or integer() or list(term()) +``` + +Warnings are also emitted if you pass a data type that does not implement the `Enumerable` protocol as a generator to for-comprehensions: + +```elixir +defmodule Example do + def my_code(%Date{} = date) do + for(x <- date, do: x) + end +end +``` + +will emit: + +``` +warning: incompatible value given to for-comprehension: + + x <- date + +it has type: + + %Date{year: term(), month: term(), day: term(), calendar: term()} + +but expected a type that implements the Enumerable protocol, it must be one of: + + dynamic( + %Date.Range{} or %File.Stream{} or %GenEvent.Stream{} or %HashDict{} or %HashSet{} or + %IO.Stream{} or %MapSet{} or %Range{} or %Stream{} + ) or fun() or list(term()) or non_struct_map() +``` + +### Type checking and inference of anonymous functions + +Elixir v1.19 can now type infer and type check anonymous functions. Here is a trivial example: + +```elixir +defmodule Example do + def run do + fun = fn %{} -> :map end + fun.("hello") + end +end +``` + +The example above has an obvious typing violation, as the anonymous function expects a map but a string is given. With Elixir v1.19, the following warning is now printed: + +``` + warning: incompatible types given on function application: + + fun.("hello") + + given types: + + binary() + + but function has type: + + (dynamic(map()) -> :map) + + typing violation found at: + │ + 6 │ fun.("hello") + │ ~ + │ + └─ mod.exs:6:8: Example.run/0 +``` + +Function captures, such as `&String.to_integer/1`, will also propagate the type as of Elixir v1.19, arising more opportunity for Elixir's type system to catch bugs in our programs. + +### Acknowledgements + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/), [Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +## Faster compile times in large projects + +This release includes two compiler improvements that can lead up to 4x faster builds in large codebases. + +While Elixir has always compiled the given files in project or a dependency in parallel, the compiler would sometimes be unable to use all of the machine resources efficiently. This release addresses two common limitations, delivering performance improvements that scale with codebase size and available CPU cores. + +### Code loading bottlenecks + +Prior to this release, Elixir would load modules as soon as they were defined. However, because the Erlang part of code loading happens within a single process (the code server), this would make it a bottleneck, reducing parallelization, especially on large projects. + +This release makes it so modules are loaded lazily. This reduces the pressure on the code server and the amount of work during compilation, with reports of more than two times faster compilation for large projects. The benefits depend on the codebase size and the number of CPU cores available. + +Implementation wise, [the parallel compiler already acts as a mechanism to resolve modules during compilation](https://elixir-lang.org/blog/2012/04/24/a-peek-inside-elixir-s-parallel-compiler/), so we built on that. By making sure the compiler controls both module compilation and module loading, it can also better guarantee deterministic builds. + +There are two potential regressions with this approach. The first one happens if you spawn processes during compilation which invoke other modules defined within the same project. For example: + +```elixir +defmodule MyLib.SomeModule do + list = [...] + + Task.async_stream(list, fn item -> + MyLib.SomeOtherModule.do_something(item) + end) +end +``` + +Because the spawned process is not visible to the compiler, it won't be able to load `MyLib.SomeOtherModule`. You have two options, either use [`Kernel.ParallelCompiler.pmap/2`](https://hexdocs.pm/elixir/Kernel.ParallelCompiler.html#pmap/2) or explicitly call [`Code.ensure_compiled!(MyLib.SomeOtherModule)`](https://hexdocs.pm/elixir/Code.html#ensure_compiled!/1) before spawning the process that uses said module. + +The second one is related to `@on_load` callbacks (typically used for [NIFs](https://www.erlang.org/doc/system/nif.html)) that invoke other modules defined within the same project. For example: + +```elixir +defmodule MyLib.SomeModule do + @on_load :init + + def init do + MyLib.AnotherModule.do_something() + end + + def something_else do + ... + end +end + +MyLib.SomeModule.something_else() +``` + +The reason this fails is because `@on_load` callbacks are invoked within the code server and therefore they have limited ability to load additional modules. It is generally advisable to limit invocation of external modules during `@on_load` callbacks but, in case it is strictly necessary, you can set `@compile {:autoload, true}` in the invoked module to address this issue in a forward and backwards compatible manner. + +Both snippets above could actually lead to non-deterministic compilation failures in the past, and as a result of these changes, compiling these cases are now deterministic. + +### Parallel compilation of dependencies + +This release introduces a variable called `MIX_OS_DEPS_COMPILE_PARTITION_COUNT`, which instructs `mix deps.compile` to compile dependencies in parallel. + +While fetching dependencies and compiling individual Elixir dependencies already happened in parallel, as outlined in the previous section, there were pathological cases where performance gains would be left on the table, such as when compiling dependencies with native code or dependencies where one or two large files would take most of the compilation time. + +By setting `MIX_OS_DEPS_COMPILE_PARTITION_COUNT` to a number greater than 1, Mix will now compile multiple dependencies at the same time, using separate OS processes. Empirical testing shows that setting it to half of the number of cores on your machine is enough to maximize resource usage. The exact speed up will depend on the number of dependencies and the number of machine cores and some users reported up to 4x faster compilation times when using our release candidates. If you plan to enable it on CI or build servers, keep in mind it will most likely have a direct impact on memory usage too. + +## Erlang/OTP 28 support + +Elixir v1.19 officially supports Erlang/OTP 28.1+ and later. In order to support the new Erlang/OTP 28 representation for regular expressions, structs can now control how they are escaped into abstract syntax trees by defining a `__escape__/1` callback. + +On the other hand, the new representation for regular expressions in Erlang/OTP 28+ implies they can no longer be used as default values for struct fields. Therefore, this is not allowed: + +```elixir +defmodule Foo do + defstruct regex: ~r/foo/ +end +``` + +You can, however, still use regexes when initializing the structs themselves: + +```elixir +defmodule Foo do + defstruct [:regex] + + def new do + %Foo{regex: ~r/foo/} + end +end +``` + +## OpenChain certification + +Elixir v1.19 is also our first release following OpenChain compliance, [as previously announced](https://elixir-lang.org/blog/2025/02/26/elixir-openchain-certification/). In a nutshell: + + * Elixir releases now include a Source SBoM in CycloneDX 1.6 or later and SPDX 2.3 or later formats. + * Each release is attested along with the Source SBoM. + +These additions offer greater transparency into the components and licenses of each release, supporting more rigorous supply chain requirements. + +This work was performed by [Jonatan Männchen](https://maennchen.dev) and sponsored by the [Erlang Ecosystem Foundation](https://erlef.org). + +## Summary + +There are many other goodies in this release, such as improved option parsing, better debuggability and performance in ExUnit, the addition of `mix help Mod`, `mix help Mod.fun`, `mix help Mod.fun/arity`, and `mix help app:package` to make documentation accessible via shell for humans and agents, and much more. [See the CHANGELOG](https://hexdocs.pm/elixir/1.19/changelog.html) for the complete release notes. + +Happy coding! diff --git a/_posts/2025-12-02-lazier-bdds-for-set-theoretic-types.markdown b/_posts/2025-12-02-lazier-bdds-for-set-theoretic-types.markdown new file mode 100644 index 000000000..01a1718ec --- /dev/null +++ b/_posts/2025-12-02-lazier-bdds-for-set-theoretic-types.markdown @@ -0,0 +1,326 @@ +--- +layout: post +title: "Lazier Binary Decision Diagrams (BDDs) for set-theoretic types" +authors: +- José Valim +- Guillaume Duboc +category: Internals +excerpt: "This article explores the data structures used to represent set-theoretic types and the recent optimizations we have applied to them" +--- + +[The Elixir team and the CNRS are working on a set-theoretic type system for Elixir](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/) which, simply put, is a type-system powered by unions, intersections, and negations. As part of the implementation of said type systems, we need an efficient way of representing said operations. This article discusses the existing approaches found in theory and practice, as well as the improvements we have introduced as part of [Elixir v1.19](/blog/2025/10/16/elixir-v1-19-0-released.markdown/). + +This article covers the implementation details of the type system. You don't need to understand these internals to use the type system, just as you don't need to know virtual machine bytecodes or compiler passes to use a programming language. Our goal is to document our progress and provide guidance for future maintainers and implementers. Let's get started. + +## DNFs - Disjunctive Normal Forms + +A Disjunctive Normal Form (DNF) is a standardized way of expressing logical formulas using only disjunctions (unions) of conjunctions (intersections). In the context of set-theoretic type systems, DNFs provide a canonical representation for union and intersection types, represented respectively as `or` and `and` in Elixir. + +In Elixir, we would represent those as lists of lists. Consider a type expression like `(A and B) or (C and D)`. This is already in DNF, it's a union of intersections, and it would be represented as: `[[A, B], [C, D]]`. This means performing unions between two DNFs is a simple list concatenation: + +```elixir +def union(dnf1, dnf2), do: dnf1 ++ dnf2 +``` + +However, more complex expressions like `A and (B or C)` need to be converted. Using distributive laws, this becomes `(A and B) or (A and C)`, which is now in DNF. In other words, the intersection of DNFs is a Cartesian product: + +```elixir +def intersection(dnf1, dnf2) do + for intersections1 <- dnf1, + intersections2 <- dnf2 do + intersections1 ++ intersections2 + end +end +``` + +The advantage of DNFs is their simple structure. Every type can be represented as unions of intersecting terms, making operations like checking if a type is empty simply a matter of checking if all unions have at least one intersection that is empty: + +```elixir +def empty?(dnf) do + Enum.all?(dnf, fn intersections -> + Enum.any?(intersections, &empty_component?/1) + end) +end +``` + + +On the other hand, the snippets above already help us build an intuition on the drawbacks of DNFs. + +First, we have seen how intersections are Cartesian products, which can lead to exponential blow ups when performing the intersection of unions. For example, `(A₁ or A₂) and (B₁ or B₂) and (C₁ or C₂)` leads to `(A₁ and B₁ and C₁) or (A₁ and B₁ and C₂) or (A₁ and B₂ and C₁) or ...`, with 8 distinct unions. + +Furthermore, if we implement unions as simple list concatenations, those unions can end up with duplicated entries, which exacerbates the exponential blow up when we perform intersections of these unions. This forces us to aggressively remove duplicates in unions, making it more complex and expensive than a concatenation. + +Despite their limitations, DNFs served us well and were the data structure used as part of Elixir v1.17 and v1.18. However, since Elixir v1.19 introduced type inference of anonymous functions, negations became more prevalent in the type system, making exponential growth more frequent. Let's understand why. + +## Inferring anonymous functions + +Imagine the following anonymous function: + +```elixir +fn + %{full_name: full} -> "#{full}" + %{first_name: first, last_name: last} -> "#{last}, #{first}" +end +``` + +We can say the first clause accepts any map with the key `full_name`. The second clause accepts any map with the keys `first_name` and `last_name` which DO NOT have the key `full_name` (otherwise they would have matched the first clause). Therefore, the inferred type should be: + +```elixir +$ %{full_name: String.Chars.t()} -> String.t() +$ %{first_name: String.Chars.t(), last_name: String.Chars.t()} and not + %{full_name: String.Chars.t()} -> String.t() +``` + +As you can see, in order to express this type, we need a negation (`not`). Or, more precisely, a difference since `A and not B` is the same as `A - B`. + +Implementing negations/differences in DNFs is relatively straightforward. Instead of lists of lists, we now use lists of two-element tuples, where the first element is a list of positive types, and the second is a list of negative types. For example, previously we said `(A and B) or (C and D)` would be represented as `[[A, B], [C, D]]`, now it will be represented as: + +```elixir +[{[A, B], []}, {[C, D], []}] +``` + +While `(A and not B) or C or D` is represented as: + +```elixir +[{[A], [B]}, {[C], []}, {[D], []}] +``` + +The difference between two DNFs is implemented similarly to intersections, except we now need to perform the Cartesian product over the positive and negative parts of each conjunction. And given anonymous functions have differences, inferring the types of anonymous functions are now exponentially expensive, which caused some projects to take minutes to compile. Not good! + +## BDDs - Binary Decision Diagrams + +Luckily, those exact issues are well documented in literature and are addressed by Binary Decision Diagrams (BDDs), introduced by [Alain Frisch (2004)](https://www.cduce.org/papers/frisch_phd.pdf) and later recalled and expanded by [Giuseppe Castagna (2016)](https://www.irif.fr/~gc/papers/covcon-again.pdf). + +BDDs represent set-theoretic operations as an ordered tree. This requires us to provide an order, any order, across all types. Given [all Elixir values have a total order](https://hexdocs.pm/elixir/Kernel.html#module-term-ordering), that's quite straightforward. Furthermore, by ordering it, we can detect duplicates as we introduce nodes in the tree. The tree can have three distinct node types: + +```elixir +type bdd() = :top or :bottom or {type(), constrained :: bdd(), dual :: bdd()} +``` + +`:top` represents the top type (where the intersection `type and :top` returns `type`) and `:bottom` represents the bottom type (where the intersection `type and :bottom` returns `:bottom`). Non-leaf nodes are represented via a three-element tuple, where the first element is the type (what we have been calling `A`, `B`... so far), the second element is called in literature the constrained branch, and the third element is the dual branch. + +In order to compute the actual type of a non-leaf node, we need to compute `(type() and constrained()) or (not type() and dual())` (hence the names constrained and dual). Let's see some examples. + +The type `A` is represented as `{A, :top, :bottom}`. This is because, if we compute `(A and :top) or (not A and :bottom)`, we get `A or :bottom`, which is equivalent to `A`. + +The type `not A` is represented as `{A, :bottom, :top}`, and it gives us `(A and :bottom) or (not A and :top)`, which yields `:bottom or not A`, which is equivalent to `not A`. + +The type `A and B`, assuming `A < B` according to a total order, is represented as `{A, {B, :top, :bottom}, :bottom}`. Expanding it node by node gives us: + +```elixir +(A and ((B and :top) or (not B and :bottom))) or (not A and :bottom) +(A and (B or :bottom)) or (not A and :bottom) +(A and B) or :bottom +(A and B) +``` + +While the difference `A and not B` is represented as `{A, {B, :bottom, :top}, :bottom}`, which we also expand node by node: + +```elixir +(A and ((B and :bottom) or (not B and :top))) or (not A and :bottom) +(A and (:bottom or not B)) or (not A and :bottom) +(A and not B) or :bottom +(A and not B) +``` + +Finally, the union `A or B` is implemented as `{A, :top, {B, :top, :bottom}}`. Let's expand it: + +```elixir +(A and :top) or (not A and ((B and :top) or (not B and :bottom))) +(A and :top) or (not A and (B or :bottom)) +A or (not A and B) +(A or not A) and (A or B) +:top and (A or B) +A or B +``` + +In other words, Binary Decision Diagrams allow us to represent unions, intersections, and differences efficiently, removing the exponential blow up. [Guillaume Duboc implemented them as part of Elixir v1.19](https://github.com/elixir-lang/elixir/pull/14693), addressing the bottlenecks introduced as part of the new type system features... but unfortunately BDDs introduced new slow downs. + +The issue with BDDs comes when applying unions to intersections and differences. Take the following type `(A and B) or C`. Since we need to preserve the order `A < B < C`, it would be represented as: + +```elixir +{A, {B, :top, {C, :top, :bottom}}, {C, :top, :bottom}} +``` + +which can be expanded as: + +```elixir +(A and ((B and :top) or (not B and ((C and :top) or (not C and :bottom))))) or (not A and ((C and :top) or (not C and :bottom))) +(A and (B or (not B and C))) or (not A and C) +(A and (B or C)) or (not A and C) +(A and B) or (A and C) or (not A and C) +(A and B) or C +``` + +As you can see, although the representation is correct, its expansion ends-up generating too many disjunctions. And while we can simplify them back to `(A and B) or C` symbolically, doing such simplications in practice are too expensive. + +In other words, the BDD expansion grows exponentially in size on consecutive unions, which is particularly troublesome because we must expand the BDD every time we check for emptiness or subtyping. + +At the end of the day, it seems we traded faster intersections/differences for slower unions. Perhaps we can have our cake and eat it too? + +## BDDs with lazy unions (or ternary decision diagrams) + +Luckily, the issue above was also forecast by Alain Frisch (2004), where he suggests an additional representation, called BDDs with lazy unions. + +In a nutshell, we introduce a new element, called `uncertain`, to each non-leaf node to represent unions: + +```elixir +type lazy_bdd() = :top or :bottom or + {type(), constrained :: lazy_bdd(), uncertain :: lazy_bdd(), dual :: lazy_bdd()} +``` + +We'll refer to the `uncertain` as unions going forward. + +The type of each non-leaf node can be computed by `(type() and constrained()) or uncertain() or (not type() and dual())`. Here are some examples: + +```elixir +A = {A, :top, :bottom, :bottom} +A and B = {A, {B, :top, :bottom, :bottom}, :bottom, :bottom} +A or B = {A, :top, {B, :top, :bottom, :bottom}, :bottom} +``` + +And, going back to `(A and B) or C`, it can be represented as: + +```elixir +{A, {B, :top, :bottom, :bottom}, {C, :top, :bottom, :bottom}, :bottom} +``` + +The duplication of `C` is fully removed. With our new representation in hand, the next step is to implement union, intersection, and difference of lazy BDDs, using the formulas found in literature and described below. + +Assuming that a lazy BDD `B` is represented as `{a, C, U, D}`, and therefore `B1 = {a1, C1, U1, D2}` and `B2 = {a2, C2, U2, D2}`, the union of the lazy BDDs `B1 or B2` can be computed as: + +```elixir +{a1, C1 or C2, U1 or U2, D1 or D2} when a1 == a2 +{a1, C1, U1 or B2, D1} when a1 < a2 +{a2, C2, B1 or U2, D2} when a1 > a2 +``` + +The intersection `B1 and B2` is: + +```elixir +{a1, (C1 or U1) and (C2 or U2), :bottom, (D1 or U1) and (D2 or U2)} when a1 == a2 +{a1, C1 and B2, U1 and B2, D1 and B2} when a1 < a2 +{a2, B1 and C2, B1 and U2, B1 and D2} when a1 > a2 +``` + +The difference `B1 and not B2` is: + +```elixir +{a1, (C1 or U1) and not (C2 or U2), :bottom, (D1 or U1) and not (D2 or U2)} when a1 == a2 +{a1, (C1 or U1) and not B2, :bottom, (D1 or U1) and not B2} when a1 < a2 +{a2, B1 and not (C2 or U2), :bottom, B1 and not (D2 or U2)} when a1 > a2 +``` + +[Guillaume Duboc first implemented lazy BDDs to represent our function types](https://github.com/elixir-lang/elixir/pull/14799), addressing some of the bottlenecks introduced alongside BDDs. Afterwards, we attempted to convert all types to use lazy BDDs, hoping they would address the remaining bottlenecks, but that was not the case. There were still some projects that type checked instantaneously in Elixir v1.18 (which used DNFs) but took minutes on v1.19 release candidates, which could only point to large unions still being the root cause. However, weren't lazy BDDs meant to address the issue with unions? + +That was the question ringing in Guillaume's head and in mine after an hours-long conversation, when we decided to call it a day. Unbeknownst to each other, we both continued working on the problem that night and the following morning. Separately, we were both able to spot the issue and converge on the same solution. + +## Lazier BDDs (for intersections) + +If you carefully look at the formulas above, you can see that intersections and differences of equal nodes cause a distribution of unions. Here is the intersection: + +```elixir +{a1, (C1 or U1) and (C2 or U2), :bottom, (D1 or U1) and (D2 or U2)} when a1 == a2 +``` + +Notice how U1 and U2 now appear on both constrained and dual parts and the whole union part of the node disappeared, now listed simply as `:bottom`. + +In addition, considering the common case where `C1 = C2 = :top` and `D1 = D2 = :bottom`, the node above becomes `{a1, :top, :bottom, U1 and U2}`, which effectively moves the unions to the dual part. If you play close attention to it, since the uncertain is now `:bottom`, we reverted back to the original BDD representation. Any further `union` on those nodes will behave exactly as in the non-lazy BDDs, which we know to be problematic. + +In other words, certain operations on lazy BDDs cause unions to revert to the previous BDD representation. So it seems lazy BDDs are not lazy enough? Could we stop this from happening? + +Guillaume and I arrived at a new formula using different approaches. Given Guillaume's approach can also be used to optimize differences, that's the one I will show below. In particular, we know the intersection of equal nodes is implemented as: + +```elixir +{a1, (C1 or U1) and (C2 or U2), :bottom, (D1 or U1) and (D2 or U2)} when a1 == a2 +``` + +If we distribute the intersection in the constrained part, we get: + +```elixir +(C1 and C2) or (C1 and U2) or (U1 and C2) or (U1 and U2) +``` + +If we distribute the intersection in the dual part, we get: + +```elixir +(D1 and D2) or (D1 and U2) or (U1 and D2) or (U1 and U2) +``` + +We can clearly see both parts have `U1 and U2`, which we can then move to the union! Leaving us with: + +```elixir +{a1, + (C1 and C2) or (C1 and U2) or (U1 and C2), + (U1 and U2), + (D1 and D2) or (D1 and U2) or (U1 and D2)} when a1 == a2 +``` + +We can then factor out `C1` in the constrained and `D1` in the dual (or `C2` and `D2` respectively), resulting in: + +```elixir +{a1, + (C1 and (C2 or U2)) or (U1 and C2), + (U1 and U2), + (D1 and (D2 or U2)) or (U1 and D2)} when a1 == a2 +``` + +While this new formula requires more operations, if we consider the common case `C1 = C2 = :top` and `D1 = D2 = :bottom`, we now have `{a1, :top, U1 and U2, :bottom}`, with the unions perfectly preserved in the middle. We independently implemented this formula and noticed it addressed all remaining bottlenecks! + +## Lazier BDDs (for differences) + +The issues we outlined above for intersections are even worse for differences. Let's check the difference formula: + +```elixir +{a1, (C1 or U1) and not (C2 or U2), :bottom, (D1 or U1) and not (D2 or U2)} when a1 == a2 +{a1, (C1 or U1) and not B2, :bottom, (D1 or U1) and not B2} when a1 < a2 +{a2, B1 and not (C2 or U2), :bottom, B1 and not (D2 or U2)} when a1 > a2 +``` + +As you can see, all operations shuffle the union nodes and return `:bottom`. But this time, we know how to improve it! Let's start with `a1 == a2`. If we expand the difference in the constrained part, we get: + +```elixir +(C1 and not C2 and not U2) or (U1 and not C2 and not U2) +``` + +If we do the same in the dual part, we have: + +```elixir +(D1 and not D2 and not U2) or (U1 and not D2 and not U2) +``` + +Unfortunately, there are no shared union terms between the constrained and dual parts, unless C2 and D2 are `:bottom`. Therefore, instead of fully rewriting the difference of equal nodes, we add the following special case: + +```elixir +{a1, C1 and not U2, U1 and not U2, D1 and not U2} +when a1 == a2 and C2 == :bottom and D2 == :bottom +``` + +We can apply a similar optimization when `a1 < a2`. The current formula: + +```elixir +{a1, (C1 or U1) and not B2, :bottom, (D1 or U1) and not B2} when a1 < a2 +``` + +The constrained part can be written as `(C1 and not B2) or (U1 and not B2)` and the dual part as `(D1 and not B2) or (U1 and not B2)`. Given `(U1 and not B2)` is shared on both parts, we can also convert it to a union, resulting in: + +```elixir +{a1, C1 and not B2, U1 and not B2, D1 and not B2} when a1 < a2 +``` + +Unfortunately, we can't apply this for `a2 > a1`, as differences are asymmetric and do not distribute over unions on the right side. Therefore, the updated formula for difference is: + +```elixir +{a1, C1 and not U2, U1 and not U2, D1 and not U2} when a1 == a2 and C2 == :bottom and D2 == :bottom +{a1, (C1 or U1) and not (C2 or U2), :bottom, (D1 or U1) and not (D2 or U2)} when a1 == a2 +{a1, C1 and not B2, U1 and not B2, D1 and not B2} when a1 < a2 +{a2, B1 and not (C2 or U2), :bottom, B1 and not (D2 or U2)} when a1 > a2 +``` + +With these new formulas, all new typing features in Elixir v1.19 perform efficiently and most projects now type check faster than in Elixir v1.18. We have also been able to use the rules above to derive additional optimizations for differences, such as when `a1 == a2 and U2 == :bottom`, which will be part of future releases. Hooray! + +## Acknowledgements + +As there is an increasing interest in implementing set-theoretic types for other dynamic languages, we hope this article shines a brief light on the journey and advancements made by the research and Elixir teams when it comes to representing set-theoretic types. + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/) and [Tidewave](https://tidewave.ai/). \ No newline at end of file diff --git a/atom.xml b/atom.xml index ec3016b38..052b63a8e 100644 --- a/atom.xml +++ b/atom.xml @@ -4,7 +4,7 @@ elixir_url: http://elixir-lang.org --- - + Elixir Lang @@ -15,15 +15,15 @@ elixir_url: http://elixir-lang.org {{ post.title }} - {% if post.author %} - - {{ post.author }} - - {% endif %} + {% for author in post.authors %} + + {{ author }} + + {% endfor %} {{ post.date | date_to_xmlschema }} {{ post.id }} {{ post.content | xml_escape }} {% endfor %} - + diff --git a/blog/categories.html b/blog/categories.html index 51192802c..53589af04 100644 --- a/blog/categories.html +++ b/blog/categories.html @@ -7,7 +7,8 @@

Posts by category:

    - {% for category in site.categories %} + {% assign sorted_cats = site.categories | sort %} + {% for category in sorted_cats %}
  • » {{ category | first | capitalize }}
      diff --git a/blog/index.html b/blog/index.html index 5e8c7624e..75d152e14 100644 --- a/blog/index.html +++ b/blog/index.html @@ -9,11 +9,11 @@

      {{ post.title }}

      -
      {{ post.date | date: "%B %d, %Y" }}{% if post.author %} · by {{ post.author}}{% endif %} . in {% for category in post.categories %}{{ category }}{% endfor %}
      +
      {{ post.date | date: "%B %d, %Y" }}{% if post.authors %} · by {{ post.authors | join: ", " }}{% endif %} · in {% for category in post.categories %}{{ category }}{% endfor %}
      - {{ post.excerpt }} +

      {{ post.excerpt }}

      Continue reading →

      diff --git a/cases.html b/cases.html new file mode 100644 index 000000000..42d49e97a --- /dev/null +++ b/cases.html @@ -0,0 +1,47 @@ +--- +section: cases +layout: default +image: /images/social/elixir-og-card.jpg +--- + +

      Cases

      + +

      Click on the cases below to learn more about how companies across different industries are using the power of Elixir and its ecosystem to create and grow their businesses. Cases are listed in the order they have been published.

      + +
      + {% assign sorted_posts = site.categories["Elixir in Production"] | sort: 'date' | reverse %} + {% for post in sorted_posts %} + {%- if post.flagship %} + + {% if post.logo == nil %} +
      + {% else %} +
      + {% endif %} +
      + {% for tag in post.tags %} + #{{tag}} + {% endfor %} +
      +       + {%- endif -%} + {% endfor %} + {% for post in sorted_posts %} + {%- unless post.flagship %} + + {% if post.logo == nil %} +
      + {% else %} +
      + {% endif %} +
      + {% for tag in post.tags %} + #{{tag}} + {% endfor %} +
      +       + {%- endunless -%} + {% endfor %} +
      + +

      diff --git a/crash-course.markdown b/crash-course.markdown index 7fdbb3b0f..acbb6da09 100644 --- a/crash-course.markdown +++ b/crash-course.markdown @@ -2,6 +2,7 @@ title: "Erlang/Elixir Syntax: A Crash Course" section: home layout: default +image: /images/social/elixir-og-card.jpg --- # {{ page.title }} @@ -28,7 +29,7 @@ hello() -> Add your functions to it, save it to disk, run `erl` from the same directory and execute the `compile` command: ```erl -Eshell V5.9 (abort with ^G) +Eshell V13.0.4 (abort with ^G) 1> c(module_name). ok 1> module_name:hello(). @@ -46,30 +47,37 @@ Elixir too has an interactive shell called `iex`. Compiling Elixir code can be d # module_name.ex defmodule ModuleName do def hello do - IO.puts "Hello World" + IO.puts("Hello world!") end end ``` And compiled from `iex`: -```iex -Interactive Elixir +```elixir +Interactive Elixir (1.14.0) - press Ctrl+C to exit (type h() ENTER for help) iex> c("module_name.ex") [ModuleName] -iex> ModuleName.hello +iex> ModuleName.hello() Hello world! :ok ``` -However notice that in Elixir you don't need to create a file only to create a new module, Elixir modules can be defined directly in the shell: +However, notice that in Elixir you don't need to create a file only to create a new module; Elixir modules can be defined directly in the shell: ```elixir -defmodule MyModule do - def hello do - IO.puts "Another Hello" - end -end +iex> defmodule MyModule do +...> def hello do +...> IO.puts("Another Hello") +...> end +...> end +{:module, MyModule, + <<70, 79, 82, 49, 0, 0, 5, 136, 66, 69, 65, 77, 65, 116, 85, 56, 0, 0, 0, 187, + 0, 0, 0, 19, 15, 69, 108, 105, 120, 105, 114, 46, 77, 121, 77, 111, 100, 117, + 108, 101, 8, 95, 95, 105, 110, 102, 111, ...>>, {:hello, 0}} +iex> MyModule.hello() +Another Hello +:ok ``` @@ -79,18 +87,19 @@ This section goes over some of the syntactic differences between the two languag ### Operator names -Some operators are spelled differently. +Some operators are spelled differently, others are not available. -| Erlang | Elixir | Meaning | -|----------------|----------------|-----------------------------------------| -| and | NOT AVAILABLE | Logical 'and', evaluates both arguments | -| andalso | and | Logical 'and', short-circuits | -| or | NOT AVAILABLE | Logical 'or', evaluates both arguments | -| orelse | or | Logical 'or', short-circuits | -| =:= | === | A match operator | -| =/= | !== | A negative match | -| /= | != | Not equals | -| =< | <= | Less than or equals | +| Erlang | Elixir | Meaning | +|----------------|----------------|--------------------------------------------------| +| and | NOT AVAILABLE | Strictly boolean 'and', evaluates both arguments | +| andalso | and | Strictly boolean "and", short-circuits | +| or | NOT AVAILABLE | Strictly boolean 'or', evaluates both arguments | +| orelse | or | Strictly boolean "or", short-circuits | +| xor | NOT AVAILABLE | Strictly boolean 'xor' | +| =:= | === | Strictly equals to | +| =/= | !== | Strictly not equals to | +| /= | != | Not equals to | +| =< | <= | Less than or equals to | ### Delimiters @@ -141,7 +150,7 @@ ok **Elixir** -```iex +```elixir iex> a = 1 1 iex> a = 2 @@ -152,20 +161,13 @@ iex> ^a = 3 ### Calling functions -Elixir allows you to omit parentheses in function calls, Erlang does not. - -| Erlang | Elixir | -|-------------------|----------------| -| some_function(). | some_function | -| sum(A, B) | sum a, b | - Invoking a function from a module uses different syntax. In Erlang, you would write ```erlang lists:last([1, 2]). ``` -to invoke the `last` function from the `List` module. In Elixir, use the dot `.` in place of the colon `:` +to invoke the `last` function from the `lists` module. In Elixir, use the dot `.` in place of the colon `:` ```elixir List.last([1, 2]) @@ -174,7 +176,7 @@ List.last([1, 2]) **Note**. Since Erlang modules are represented by atoms, you may invoke Erlang functions in Elixir as follows: ```elixir -:lists.sort [3, 2, 1] +:lists.sort([3, 2, 1]) ``` All of the Erlang built-ins reside in the `:erlang` module. @@ -224,11 +226,11 @@ is_atom(''). %=> true **Elixir** ```elixir -is_atom :ok #=> true -is_atom :'ok' #=> true -is_atom Ok #=> true -is_atom :"Multiple words" #=> true -is_atom :"" #=> true +is_atom(:ok) #=> true +is_atom(:'ok') #=> true +is_atom(Ok) #=> true +is_atom(:"Multiple words") #=> true +is_atom(:"") #=> true ``` ### Tuples @@ -269,22 +271,22 @@ is_binary(<<"Hello">>). %=> true **Elixir** ```elixir -is_list 'Hello' #=> true -is_binary "Hello" #=> true -is_binary <<"Hello">> #=> true +is_list('Hello') #=> true +is_binary("Hello") #=> true +is_binary(<<"Hello">>) #=> true <<"Hello">> === "Hello" #=> true ``` -In Elixir, the word **string** means a UTF-8 binary and there is a `String` module that works on such data. Elixir also expects your source files to be UTF-8 encoded. On the other hand, **string** in Erlang refers to char lists and there is a `:string` module, that's not UTF-8 aware and works mostly with char lists. +In Elixir, the word **string** means a UTF-8 binary and there is a `String` module that works on such data. Elixir also expects your source files to be UTF-8 encoded. On the other hand, **string** in Erlang refers to char lists and there is a `:string` module that works mostly with both char lists and UTF-8 encoded binaries. Elixir also supports multiline strings (also called *heredocs*): ```elixir -is_binary """ +is_binary(""" This is a binary spanning several lines. -""" +""") #=> true ``` @@ -310,7 +312,7 @@ kw[:another_key] ### Maps -Erlang R17 introduced maps, a key-value store, with no ordering. Keys and values can be any term. Creating, updating and matching maps in both languages is shown below: +Maps are key-value pairs with no ordering. Keys and values can be any term. Creating, updating and matching maps in both languages is shown below: **Erlang** @@ -355,22 +357,10 @@ re:run("abc ", Pattern). **Elixir** ```elixir -Regex.run ~r/abc\s/, "abc " +Regex.run(~r/abc\s/, "abc ") #=> ["abc "] ``` -Regexes are also supported in heredocs, which is convenient when defining multiline regexes: - -```elixir -Regex.regex? ~r""" -This is a regex -spanning several -lines. -""" -#=> true -``` - - ## Modules Each Erlang module lives in its own file which has the following structure: @@ -400,12 +390,12 @@ An Elixir equivalent to the Erlang above: defmodule HelloModule do # A "Hello world" function def some_fun do - IO.puts "Hello world!" + IO.puts("Hello world!") end # This one works only with lists def some_fun(list) when is_list(list) do - IO.inspect list + IO.inspect(list) end # A private function @@ -421,7 +411,7 @@ In Elixir, it is also possible to have multiple modules in one file, as well as defmodule HelloModule do defmodule Utils do def util do - IO.puts "Utilize" + IO.puts("Utilize") end defp priv do @@ -437,20 +427,20 @@ end defmodule ByeModule do end -HelloModule.dummy +HelloModule.dummy() #=> :ok -HelloModule.Utils.util -#=> "Utilize" +HelloModule.Utils.util() +# "Utilize" +#=> :ok -HelloModule.Utils.priv +HelloModule.Utils.priv() #=> ** (UndefinedFunctionError) undefined function: HelloModule.Utils.priv/0 ``` - ## Function syntax -[This chapter][3] from the Erlang book provides a detailed description of pattern matching and function syntax in Erlang. Here, I'm briefly covering the main points and provide sample code both in Erlang and Elixir. +[This chapter][3] from the Erlang book provides a detailed description of pattern matching and function syntax in Erlang. Here, we briefly cover the main points and provide sample code both in Erlang and Elixir. [3]: http://learnyousomeerlang.com/syntax-in-functions @@ -461,9 +451,9 @@ Pattern matching in Elixir is based on Erlang's implementation and in general is **Erlang** ```erlang -loop_through([H | T]) -> - io:format('~p~n', [H]), - loop_through(T); +loop_through([Head | Tail]) -> + io:format('~p~n', [Head]), + loop_through(Tail); loop_through([]) -> ok. @@ -472,9 +462,9 @@ loop_through([]) -> **Elixir** ```elixir -def loop_through([h | t]) do - IO.inspect h - loop_through t +def loop_through([head | tail]) do + IO.inspect(head) + loop_through(tail) end def loop_through([]) do @@ -528,8 +518,8 @@ sum(1, 2). sum([1], [2]). %=> [1, 2] -sum("a", "b"). -%=> "ab" +sum(<<"a">>, <<"b">>). +%=> <<"ab">> ``` **Elixir** @@ -547,13 +537,13 @@ def sum(a, b) when is_binary(a) and is_binary(b) do a <> b end -sum 1, 2 +sum(1, 2) #=> 3 -sum [1], [2] +sum([1], [2]) #=> [1, 2] -sum "a", "b" +sum("a", "b") #=> "ab" ``` @@ -566,8 +556,8 @@ def mul_by(x, n \\ 2) do x * n end -mul_by 4, 3 #=> 12 -mul_by 4 #=> 8 +mul_by(4, 3) #=> 12 +mul_by(4) #=> 8 ``` ### Anonymous functions @@ -589,12 +579,12 @@ lists:map(Square, [1, 2, 3, 4]). **Elixir** ```elixir -sum = fn(a, b) -> a + b end +sum = fn a, b -> a + b end sum.(4, 3) #=> 7 -square = fn(x) -> x * x end -Enum.map [1, 2, 3, 4], square +square = fn x -> x * x end +Enum.map([1, 2, 3, 4], square) #=> [1, 4, 9, 16] ``` @@ -620,11 +610,12 @@ F({a, b}). ```elixir f = fn - {:a, :b} = tuple -> - IO.puts "All your #{inspect tuple} are belong to us" - [] -> - "Empty" - end + {:a, :b} = tuple -> + "All your #{inspect(tuple)} are belong to us" + + [] -> + "Empty" +end f.([]) #=> "Empty" @@ -641,13 +632,19 @@ Anonymous functions are first-class values, so they can be passed as arguments t **Erlang** ```erlang +% math.erl -module(math). -export([square/1]). square(X) -> X * X. +``` -lists:map(fun math:square/1, [1, 2, 3]). -%=> [1, 4, 9] +```erl +Eshell V5.9 (abort with ^G) +1> c(math). +{ok,math} +2> lists:map(fun math:square/1, [1, 2, 3]). +[1,4,9] ``` **Elixir** @@ -659,7 +656,7 @@ defmodule Math do end end -Enum.map [1, 2, 3], &Math.square/1 +Enum.map([1, 2, 3], &Math.square/1) #=> [1, 4, 9] ``` @@ -669,10 +666,10 @@ Enum.map [1, 2, 3], &Math.square/1 Elixir supports partial application of functions which can be used to define anonymous functions in a concise way: ```elixir -Enum.map [1, 2, 3, 4], &(&1 * 2) +Enum.map([1, 2, 3, 4], &(&1 * 2)) #=> [2, 4, 6, 8] -List.foldl [1, 2, 3, 4], 0, &(&1 + &2) +List.foldl([1, 2, 3, 4], 0, &(&1 + &2)) #=> 10 ``` @@ -685,7 +682,7 @@ defmodule Math do end end -Enum.map [1, 2, 3], &Math.square/1 +Enum.map([1, 2, 3], &Math.square/1) #=> [1, 4, 9] ``` @@ -749,14 +746,17 @@ Test_fun(10). **Elixir** ```elixir -test_fun = fn(x) -> +test_fun = fn x -> cond do x > 10 -> :greater_than_ten + x < 10 and x > 0 -> :less_than_ten_positive + x < 0 or x === 0 -> :zero_or_negative + true -> :exactly_ten end @@ -810,9 +810,9 @@ end. **Elixir** ```elixir -pid = Kernel.self +pid = Kernel.self() -send pid, {:hello} +send(pid, {:hello}) receive do {:hello} -> :ok @@ -822,7 +822,6 @@ after end ``` - ## Adding Elixir to existing Erlang programs Elixir compiles into BEAM byte code (via Erlang Abstract Format). This means that Elixir code can be called from Erlang and vice versa, without the need to write any bindings. All Elixir modules start with the `Elixir.` prefix followed by the regular Elixir name. For example, here is how to use the UTF-8 aware `String` downcase from Elixir in Erlang: @@ -835,34 +834,6 @@ downcase(Bin) -> 'Elixir.String':downcase(Bin). ``` -### Rebar integration - -If you are using rebar, you should be able to include Elixir git repository as a dependency: - - https://github.com/elixir-lang/elixir.git - -Elixir is structured similar to Erlang's OTP. It is divided into applications that are placed inside the `lib` directory, as seen in its [source code repository](https://github.com/elixir-lang/elixir). Since rebar does not recognize such structure, we need to explicitly add to our `rebar.config` which Elixir apps we want to use, for example: - -```erlang -{lib_dirs, [ - "deps/elixir/lib" -]}. -``` - -This should be enough to invoke Elixir functions straight from your Erlang code. If you are also going to write Elixir code, you can [install Elixir's rebar plugin for automatic compilation](https://github.com/yrashk/rebar_elixir_plugin). - -### Manual integration - -If you are not using rebar, the easiest approach to use Elixir in your existing Erlang software is to install Elixir using one of the different ways specified in the [Getting Started guide](/getting-started/introduction.html) and add the `lib` directory in your checkout to `ERL_LIBS`. - - ## Further reading -Erlang's official documentation site has a nice [collection][4] of programming examples. It can be a good exercise to translate them into Elixir. [Erlang cookbook][5] offers even more useful code examples. - -Elixir also provides a [Getting Started guide][6] and has [documentation available online][7]. - -[4]: http://www.erlang.org/doc/programming_examples/users_guide.html -[5]: http://schemecookbook.org/Erlang/TOC -[6]: /getting-started/introduction.html -[7]: /docs.html +Both programming languages have plenty of resources available to dig deeper. Check out their respective websites ([Elixir](https://elixir-lang.org) and [Erlang](https://www.erlang.org)) for more information. \ No newline at end of file diff --git a/css/style.css b/css/style.css index 44a6104c7..e4d6c8839 100644 --- a/css/style.css +++ b/css/style.css @@ -33,8 +33,8 @@ html { } body { margin: 0; - font: 0.9em/1.692307em 'Bitter', Georgia, 'Times New Roman', Times, serif; - color: #555; + font: 0.95em/1.692307em 'Bitter', Georgia, 'Times New Roman', Times, serif; + color: #24292e; padding: 0; height: 100%; } @@ -50,10 +50,8 @@ a.spec, a.spec:visited { } a:hover { text-decoration: none; - color: #000; + color: #24292e; } -a:focus { outline: none; } -a:hover, a:active { outline: 0; } abbr { cursor: help } abbr[title] { border-bottom: 1px dotted; } acronym { @@ -64,7 +62,7 @@ acronym { cursor: help; } acronym:hover { - color: #333; + color: #24292e; background: #f5f5f5; border-bottom: 1px dotted #aaa; } @@ -107,13 +105,13 @@ hr { ins { font-family: Georgia, 'Times New Roman', Times, serif; background: #f5f5f5; - color: #000; + color: #24292e; text-decoration: none; font-style: italic; } mark { background: #f5f5f5; - color: #000; + color: #24292e; font-style: italic; font-weight: bold; } @@ -172,15 +170,15 @@ h1, h2, h3, h4, h5, h6 { font-style: normal; font-weight: normal; margin: 0 0 15px 0; - color: #333; + color: #24292e; line-height: 1.3em; } -h1 { font-size: 2.769230769230769em; } /* 36 / 13 = 2.769230769230769 */ +h1 { font-size: 2.5384615384615383em; } /* 33 / 13 = 2.5384615384615383 */ h2 { - font-size: 1.846153846153846em; /* 24 / 13 = 1.846153846153846 */ - margin-top: 1.458333333em; /* 35 / 24 = 1.458333333 */ + font-size: 1.6923076923076923em; /* 22 / 13 = 1.6923076923076923 */ + margin-top: 1.3333333333333333em; /* 32 / 24 = 1.3333333333333333 */ } -h3 { font-size: 1.538461538461538em; } /* 20 / 13 = 1.538461538461538 */ +h3 { font-size: 1.461538461538461em; } /* 19 / 13 = 1.538461538461538 */ h4 { font-size: 1.384615384615385em; } /* 18 / 13 = 1.384615384615385 */ h5 { font-size: 1.230769230769231em; /* 16 / 13 = 1.230769230769231 */ @@ -194,6 +192,7 @@ img { border: 0; -ms-interpolation-mode: bicubic; vertical-align: middle; + max-width: 100%; } svg:not(:root) { overflow: hidden; } form { margin: 0; } @@ -280,7 +279,6 @@ img.alignleft, img.alignright { blockquote.alignleft, blockquote .alignright { width: 33%; } .byline abbr, .entry-meta abbr, .comment-meta abbr { border: none; } .clear { clear: both; } -.no-border { border: 0 }; /* Tables -------------------------------------------------------------- */ @@ -291,7 +289,7 @@ table { table caption { font-size: 0.8125em; line-height: 1.692307em; - color: #888; + color: #666; } table th { font-size: 0.8461538461538462em; @@ -305,7 +303,7 @@ table th { } td { padding: 0.8125em 2%; - color: #888; + color: #666; border-bottom: 1px solid #e7e7e7; } @@ -314,7 +312,7 @@ td { width: auto; width: 85%; } -#content table caption { color: #555; } +#content table caption { color: #24292e; } #content table th { font-weight: bold; padding: 10px 4%; @@ -341,7 +339,7 @@ dl dt { dl dd { margin: 0 0 5px 20px; padding: 0; - color: #888; + color: #666; } /* Blockquotes @@ -352,7 +350,7 @@ blockquote { font-size: 1.153846153846154em; /* 15 / 13 = 1.153846153846154 */ font-family: Georgia, 'Times New Roman', Times, serif; font-style: italic; - color: #aaa; + color: #666; margin: 0 0 20px 0; padding: 5px 0 5px 20px; } @@ -365,13 +363,13 @@ blockquote p:last-child, blockquote pre:last-child { -------------------------------------------------------------- */ code { padding: 1px 3px; - color: #555; - background: #fff8ed; + color: #24292e; + background: #fdfbf7; border:1px solid #faefe0; } pre { padding: 15px 20px; - background: #fff8ed; + background: #fdfbf7; border: 1px solid #f6e4cc; } pre code { @@ -392,9 +390,8 @@ form label { input[type="text"], input[type="password"], input[type="email"], .input-text, textarea, select { border: 1px solid #ddd; padding: 5px; - outline: none; font-size: 0.8125em; - color: #888; + color: #666; margin: 0; display: block; background: #fff; @@ -430,7 +427,6 @@ a:hover img, #slider-nav li a:hover img { opacity: 0.85; } .hentry img, .entry-content img, .widget img { height: auto; padding: 1px; - border: 1px solid #e5e5e5; } .rss-button { margin-bottom: 25px; } iframe.video { border:0; } @@ -466,32 +462,22 @@ iframe.video { border:0; } -------------------------------------------------------------- */ #menu-primary { font-size: 1em; - width: 75%; - margin: 30px 0 12px 0; - float: right; - position: relative; - z-index: 99; } -#menu-primary .menu { - float: right; - margin-left: 0; -} -#menu-primary .menu ul { - float: right; - margin: 0; +#menu-primary ul { + margin: 0 0 10px; } #menu-primary li { - display: block; - float: left; - min-height: 30px; + display: inline-block; } #menu-primary li a { - line-height: 1em; + line-height: 1.2em; font-size: 1em; - margin-left: 20px; - display: block; + margin-right: 20px; text-transform: uppercase; } +#menu-primary li:last-child a { + margin-right: 0; +} #menu-primary li.current-menu-item a { color: #aaa; } #menu-primary ul li a:hover { color: #222; } @@ -500,6 +486,8 @@ body.install div.menu li.install a, body.getting-started div.menu li.getting-started a, body.blog div.menu li.blog a, body.docs div.menu li.docs a, +body.development div.menu li.development a, +body.cases div.menu li.cases a, body.learning div.menu li.learning a { color: #aaa; } @@ -509,61 +497,38 @@ body.learning div.menu li.learning a { #header { clear: both; width: 100%; + display: flex; + flex-wrap: wrap; + justify-content: space-between; + margin: 20px 0; } #branding { - float: left; - width: 25%; - overflow: hidden; - margin-top: 20px; -} -#site-title { - font-size: 2.307692307692308em; /* 30 / 13 = 2.307692307692308 */ - line-height: 1em; - font-weight: bold; - margin: 0 0 20px 5px; - text-transform: uppercase; - letter-spacing: -2px; - font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; - word-wrap: break-word; + max-width: 225px; + margin-bottom: 25px; } -#site-title a { - color: #222; - border-bottom: none; -} -#header img { max-width: 100%; } -#header { - padding-bottom: 10px; +.sidebar .widget.news { + font-size: 0.8em; + margin-bottom: 7px; } -#site-description { - display: none; +.sidebar .widget.events { + margin-bottom: 5px; } -#site-description { - font-size: 20px; - line-height: 1.5em; - margin: 0; - color: #333; - text-transform: none; - float: left; - width: 91.48936170212766%; - clear: both; - text-align: center; - border-top: 5px solid #444; - padding: 18px 4.25531914893617% 20px 4.25531914893617%; /* 40px / 940px = 4.25531914893617% */ - border-bottom: 1px solid #e5e5e5; - margin-bottom: 30px; +.widget.events a { + display: inline-block; + padding-bottom: 21px; + text-decoration: none; } -.sidebar .widget.news { - color: #333; - font-size: 0.8em; - margin-bottom: 7px; +.widget.events a:hover { + color: #24292e; } -.widget.news h3 a { - color: #333; +.widget.events a img { + height: auto; + max-width: 240px; } /* Posts @@ -579,12 +544,6 @@ body.learning div.menu li.learning a { .hentry:last-child { border-bottom: 0; } -.singular .hentry { - margin: 0 0 30px 0; - position: relative; - float: left; - width: 100%; -} .featured { margin-bottom: 20px; } .sticky .sticky-header { float: left; @@ -610,22 +569,83 @@ body.learning div.menu li.learning a { .hentry .entry-title { margin: 0 0 0.25em 0; padding: 0; - font-size: 1.846153846153846em; /* 24 / 13 = 1.846153846153846 */ text-transform: uppercase; line-height: 1.3em; border: none; - color: #333; word-spacing: 2px; } -.singular .entry-title { - font-size: 1.846153846153846em; /* 24 / 13 = 1.846153846153846 */ - margin-bottom: 0.625em; -} -.singular-page .entry-title { color: #ccc; } -.singular-page .entry-title, .singular-attachment .entry-title { margin-bottom: 1.2em; } .entry-title a, .entry-title a:visited { color: #222; } .entry-title a:hover { color: #4e2a5e; } +/* Cases +-------------------------------------------------------------- */ +.cases-header { + display: flex; + flex-wrap: wrap; + justify-content: space-between; +} + +.cases-boxes { + display: flex; + flex-wrap: wrap; + justify-content: space-around; +} + +.cases-box { + margin: 1em 0.25em; + text-decoration: none; + width: 205px; +} + +#shuffled-cases .cases-box { + display: none; +} + +#shuffled-cases .cases-box:nth-child(-n+3) { + display: block; +} + +.cases-image { + display: flex; + justify-content: center; + width: 175px; + height: 125px; + padding: 10px; + outline-style: solid; + outline-width: 1.5px; + outline-color: #E5E5E5; + margin: auto; + background-repeat: no-repeat; + background-position: center; + background-size: 140px; +} + +.cases-tag { + text-align: center; + padding: 10px 0; + margin: 0; +} + +.cases-tag span { + display: inline-block; + font-size: 13px; + text-decoration: none; + color: #ffffff; + padding: 2px 3px; + background-color: #9DA1A6; + border-radius: 3px; + margin: 5px 3px; +} + +.cases-box:hover .cases-image { + outline-width: 2px; + outline-color: #4E2A8E; +} + +.cases-box:hover .cases-tag span { + background-color: #4E2A8E; +} + /* Post bylines/datelines -------------------------------------------------------------- */ .byline { @@ -637,9 +657,8 @@ body.learning div.menu li.learning a { line-height: 1.692307em; word-spacing: 2px; } -.singular .byline { margin-bottom: 1.7em; } .byline a, .byline a:visited { color: #aaa; } -.byline a:hover { color: #000; } +.byline a:hover { color: #24292e; } .author, .published, .category, .edit { font-family: 'Bitter', Georgia, 'Times New Roman', Times, serif; font-style: normal; @@ -662,41 +681,32 @@ body.learning div.menu li.learning a { font-size: 0.8461538461538462em; color: #aaa; } -.entry-meta a { color: #888; } -.entry-meta a:hover { color: #000; } - -/* Singular post prev/next links --------------------------------------------------------------- */ -.singular .loop-nav { - font-size: 0.8461538461538462em; - color: #888; - clear: left; -} +.entry-meta a { color: #666; } +.entry-meta a:hover { color: #24292e; } /* Archive/search pagination and comment pagination -------------------------------------------------------------- */ -.comment-navigation { margin-bottom: 1.692307em; } .pagination.loop-pagination { text-align: center; clear: both; - margin: 7px 0; + padding: 15px 0 5px; } -.pagination .page-numbers, .comment-navigation .page-numbers { +.pagination .page-numbers { display: inline-block; padding: 4px 8px; margin: 0; line-height: 1em; color: #444; } -.pagination a.page-numbers, .comment-navigation a.page-numbers { - color: #333; +.pagination a.page-numbers { + color: #24292e; background: #e9e9e9; } -.pagination a:hover, .comment-navigation a:hover { +.pagination a:hover { color: #fff; - background: #555; + background: #000; } -.pagination .current, .comment-navigation .current { color: #aaa; } +.pagination .current { color: #aaa; } /* Widgets -------------------------------------------------------------- */ @@ -704,23 +714,16 @@ body.learning div.menu li.learning a { float: left; width: 100%; margin-bottom: 26px; - color: #888; + color: #666; } .widget table, .widget ul, .widget ol { margin: 0 0 0 16px; } -li.image { - list-style: none; - margin-bottom: 10px; -} - -.widget li.image img { border-width: 0; } - /* Widget titles -------------------------------------------------------------- */ .sidebar .widget-title { font-size: 0.7692307692307692em; /* 10 / 13 = 0.7692307692307692 */ - color: #aaa; + color: #24292e; text-transform: uppercase; letter-spacing: 1px; word-spacing: 2px; @@ -732,14 +735,14 @@ li.image { .widget .search-form label { font-size: 0.8461538461538462em; line-height: 1.692307em; - color: #aaa; + color: #666; } .widget .search-form input[type="text"] { width: 91.538461%; float: left; padding: 8px 10px; font-size: 1em; - color: #aaa; + color: #666; } .search-form .search-submit { display: none; } @@ -759,14 +762,14 @@ li.image { text-decoration: underline; } -#copyright { +#trademark { text-align: center; font-size: 12px; padding: 0 0 10px; } #edit-on-github { - padding: 15px 0 25px; + padding: 10px 0 15px; text-align: center; font-size: 12px; } @@ -791,48 +794,24 @@ li.image { } body { line-height: 1.615384615384615em; } p { margin-bottom: 1.615384615384615em; } - #branding { - float: left; - width: 100%; - position: relative; - } .hentry .archive-thumbnail { width: 110px; height: 110px; } #site-description { font-size: 1.3em; } - #menu-primary { - float: left; - clear: both; - width: 100%; - margin-top: 5px; - } - #menu-primary .menu { float: left; } - #menu-primary ul li { clear: left; } - #menu-primary ul li a { margin-left: 0; } #content { width: 100%; } .comment-list li li { padding-left: 0; } #sidebar-primary { width: 100%; clear: left; + border-top: 1px dotted #CCC; + padding-top: 45px; } - #distilled-by ul { + #created-by ul { margin: 0; } } -/* Tiny Mobile (portrait) */ -@media only screen and (max-width: 467px) { - #branding { - border-bottom: 1px solid #eee; - margin-bottom: 10px; - } - - #header img { - max-width: 100px; - } -} - /* Elixir-lang.org - Pages and Specific Elements ------------------------------------------------------ */ @@ -846,6 +825,18 @@ body.learning h4.resource { padding-top: 1em; } +body.learning .free { + background-color: #4E2A8E; + color: #ffffff; + display: inline-block; + font-size: 14px; + text-decoration: none; + padding: 0 5px; + border-radius: 5px; + line-height: 18px; + margin-left: 5px; +} + /* Jekyll Table of Contents */ ol.jekyll-toc, ol.jekyll-toc ol { @@ -896,7 +887,7 @@ ol.jekyll-toc li a { } .jekyll-toc-header a.jekyll-toc-anchor, .jekyll-toc-header a.jekyll-toc-anchor:hover { - color: #000 + color: #24292e } .jekyll-toc-header:hover a.jekyll-toc-anchor { cursor: pointer; @@ -914,7 +905,7 @@ ol.jekyll-toc li a { .jekyll-toc-header span.jekyll-toc-icon { display:none; vertical-align: middle; - color: #000; + color: #24292e; } .jekyll-toc-header:hover span.jekyll-toc-icon { display:inline-block; @@ -939,71 +930,67 @@ ol.jekyll-toc li a { user-select: none; } -/* elixir radar */ -.elixir-radar-cta { - padding: 10px 0; - display: table; - width: 100%; - border-top: 1px dashed #e5e5e5; - border-bottom: 1px dashed #e5e5e5; +/* mini documentary */ +#mini-docu a { + display: block; + margin-bottom: 12px; + text-decoration: none; } -.cta-copy { - display: table-cell; - margin-right: 10px; - vertical-align: middle; +.mini-docu-cta { + background-image: url(/images/learning/mini-docu.png); + background-repeat: no-repeat; + border: 2px solid #eee; + border-radius: 30px; + min-width: 208px; + padding: 7px 20px 4px 7px; + background-position: 15px 0px; + text-align: center; } -.cta-copy .cta-title { - font-size: 21px; - font-family: 'Bree Serif'; - color: #14485b; +.mini-docu-copy { + color: #4e2a8e; + font-family: 'Bitter'; + height: 48px; + margin-left: 66px; } -.cta-copy .cta-subtitle { - font-size: 14px; - font-family: Georgia; - color: #14485b; +.mini-docu-copy:hover { + color: black; } -.cta-button-container { - display: table-cell; - vertical-align: middle; +/* Top banner */ +#top-banner { + background: #fdfbf7; + border-bottom: 1px solid #e4d9c9; + line-height: 1; + text-align: center; + padding: 21px 30px 16px 30px; + margin-bottom: 5px; + position: relative; } -a.cta-button, a.cta-button:visited{ - color: white; - text-decoration: none; +#top-banner .close { + cursor: pointer; + position: absolute; + right: 0; + top: 0; + padding: 13px 18px; + font-size: 22px; + opacity: 0.35; } -a.cta-button { - padding: 5px 10px; - display: inline-block; - - font-size: 1.1em; - text-shadow: 0 -1px 0 rgba(0, 0, 0, .25); - text-align: center; - - background-color: #17678a; - background-image: linear-gradient(#1e88b6, #17678a); - - border: 1px solid #14485b; - border-radius: 3px; - - box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.4); +#top-banner .close:hover { + opacity: 0.7; } -a.cta-button:hover { - background-color: #10465e; - background-image: linear-gradient(#17678a, #10465e); - border-color: #0b2731; +/* Getting started */ +.getting-started-title h1 { + font-size: 2.3em; } -a.cta-button:active { - position: relative; - top: 1px; - left: 1px; - - background-image: none; - box-shadow: inset 0 2px 4px rgba(0, 0, 0, .15); -} +.getting-started-title small { + font-size: 1em; + color: #666; + display: block; +} \ No newline at end of file diff --git a/css/syntax.css b/css/syntax.css index f6f38de28..9dc212c54 100644 --- a/css/syntax.css +++ b/css/syntax.css @@ -1,8 +1,8 @@ .highlight {} .highlight .c { color: #998; font-style: italic } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ -.highlight .k { font-weight: bold } /* Keyword */ -.highlight .o { font-weight: bold } /* Operator */ +.highlight .k { color: #54585b; font-weight: bold } /* Keyword */ +.highlight .o { color: #54585b; font-weight: bold } /* Operator */ .highlight .cm { color: #998; font-style: italic } /* Comment.Multiline */ .highlight .cp { color: #999; font-weight: bold } /* Comment.Preproc */ .highlight .c1 { color: #998; font-style: italic } /* Comment.Single */ @@ -16,13 +16,13 @@ .highlight .gi .x { color: #000; background-color: #afa } /* Generic.Inserted.Specific */ .highlight .go { color: #888 } /* Generic.Output */ .highlight .gp { color: #555 } /* Generic.Prompt */ -.highlight .gs { font-weight: bold } /* Generic.Strong */ +.highlight .gs { color: #54585b; font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #aaa } /* Generic.Subheading */ .highlight .gt { color: #a00 } /* Generic.Traceback */ -.highlight .kc { font-weight: bold } /* Keyword.Constant */ -.highlight .kd { font-weight: bold } /* Keyword.Declaration */ -.highlight .kp { font-weight: bold } /* Keyword.Pseudo */ -.highlight .kr { font-weight: bold } /* Keyword.Reserved */ +.highlight .kc { color: #54585b; font-weight: bold } /* Keyword.Constant */ +.highlight .kd { color: #54585b; font-weight: bold } /* Keyword.Declaration */ +.highlight .kp { color: #54585b; font-weight: bold } /* Keyword.Pseudo */ +.highlight .kr { color: #54585b; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #458; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #099 } /* Literal.Number */ .highlight .s { color: #d14 } /* Literal.String */ @@ -36,7 +36,7 @@ .highlight .nn { color: #555 } /* Name.Namespace */ .highlight .nt { color: #000080 } /* Name.Tag */ .highlight .nv { color: #008080 } /* Name.Variable */ -.highlight .ow { font-weight: bold } /* Operator.Word */ +.highlight .ow { color: #54585b; font-weight: bold } /* Operator.Word */ .highlight .w { color: #bbb } /* Text.Whitespace */ .highlight .mf { color: #099 } /* Literal.Number.Float */ .highlight .mh { color: #099 } /* Literal.Number.Hex */ @@ -57,4 +57,4 @@ .highlight .vc { color: #008080 } /* Name.Variable.Class */ .highlight .vg { color: #008080 } /* Name.Variable.Global */ .highlight .vi { color: #008080 } /* Name.Variable.Instance */ -.highlight .il { color: #099 } /* Literal.Number.Integer.Long */ \ No newline at end of file +.highlight .il { color: #099 } /* Literal.Number.Integer.Long */ diff --git a/development.markdown b/development.markdown new file mode 100644 index 000000000..b84d2552e --- /dev/null +++ b/development.markdown @@ -0,0 +1,56 @@ +--- +title: "Development & Team" +section: development +layout: default +image: /images/social/elixir-og-card.jpg +--- + +# {{ page.title }} + +This page outlines the language's past and future development. + +## Development + +José Valim created Elixir in 2012 as a Research and Development project inside Plataformatec. Elixir's goal is to be [a productive and extensible language](/blog/2013/08/08/elixir-design-goals/) for writing maintainable and reliable software. + +Elixir runs on top of the Erlang Virtual Machine, which provides a scalable and fault-tolerant foundation. Elixir was designed to leverage this foundation without performance costs and aims to contribute to the wider ecosystem whenever possible. + +Elixir's source code is under the [Apache 2 License](https://github.com/elixir-lang/elixir/blob/main/LICENSE) and is maintained by the [Elixir Team](#team). The source code and contribution guidelines can be found on [the language repository](https://github.com/elixir-lang/elixir). + +Elixir v1.0 was released in September 2014 and a new minor version is released every 6 months, around May and November of every year. New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann) with a link to the complete CHANGELOG. All security releases [will be tagged with "[security]"](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). Security vulnerabilities should be disclosed to [elixir-security@googlegroups.com](mailto:elixir-security@googlegroups.com). Our [compatibility and deprecation policies](https://hexdocs.pm/elixir/compatibility-and-deprecations.html#content) are also documented. + +Since v1.0, the language development has become focused to provide a compact and consistent core. The Elixir team focuses on language features that: + + 1. are necessary for developing the language itself + 2. bring important concepts/features to the community in a way its effect can only be maximized or leveraged by making it part of the language + +The language development is open, both in terms of source code and of collaborations. All features and bug fixes planned for the next releases can be found [in the issues tracker](https://github.com/elixir-lang/elixir/issues). Features that may cause a larger impact on the ecosystem are first proposed to the community in [the Elixir mailing list](https://groups.google.com/group/elixir-lang-core) as well as in [the "Elixir News" section in the Elixir Forum](https://elixirforum.com/c/elixir-news). + +Community members are welcome to propose new features for Elixir. Before submitting a proposal, members are encouraged to gather feedback from around the community in whatever venues seem best. However, in order for a proposal to be considered for inclusion by the Elixir Core team, it must go through the Elixir mailing list. This often includes discussion and refinement of the proposal. The Elixir Core team has the final say on whether a proposal is accepted or rejected. While members are encouraged to gain support from the rest of the community, popularity does not mean that a proposal will be accepted. + +To remain focused, Elixir trusts its ecosystem to bring diversity and broaden its use cases. Therefore the language was designed to be extensible: the constructs available to build the language are also available for developers to extend the language and bring it to different domains. Projects such as [the Phoenix web framework](http://phoenixframework.org), [the Nerves embedded framework](http://nerves-project.org), and [Numerical Elixir](https://github.com/elixir-nx/nx) are such examples. + +Elixir also relies on a vibrant community to support its growth. The community is behind the meetups, events, learning resources, open source projects, and more. See the sidebar, the [Learning Resources](/learning.html) and [the Hex Package Manager website](https://hex.pm/) for some examples and more information. + +The best way to support the language is by getting involved in its community and contributing to the ecosystem. + +Welcome! + +## Team + +The Elixir Team is composed by: + + * José Valim + * Eric Meadows-Jönsson + * Andrea Leopardi + * Fernando Tapia Rico + * Jean Klingler + +We are also grateful to the past members of the Elixir Team: + + * Alexei Sholik + * James Fish + * Aleksei Magusev + * Michał Muskała + +Finally, we thank all of our [contributors](https://github.com/elixir-lang/elixir/graphs/contributors). diff --git a/docs.markdown b/docs.markdown index 532c3f17f..55c20390a 100644 --- a/docs.markdown +++ b/docs.markdown @@ -2,53 +2,48 @@ title: Elixir Documentation section: docs layout: default +image: /images/social/elixir-og-card.jpg --- -## Documentation +# Documentation -Choose which version you want documentation for. +The Elixir programming language is broken into 6 applications. The links below +reference the documentation for the modules and functions in each of those +applications. See also [our Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +and [the Learning page](/learning.html) for books, courses, videos, and more. -#### Stable +{% assign stable = site.data.elixir-versions[site.data.elixir-versions.stable] %} -* [Elixir](https://hexdocs.pm/elixir/) - standard library -* [EEx](https://hexdocs.pm/eex/) - templating library -* [ExUnit](https://hexdocs.pm/ex_unit/) - unit test library -* [IEx](https://hexdocs.pm/iex/) - interactive shell -* [Logger](https://hexdocs.pm/logger/) - built-in Logger -* [Mix](https://hexdocs.pm/mix/) - build tool +{% for version in site.data.elixir-versions %} + {% if version[0] == 'stable' %} + {% continue %} + {% endif %} -#### Master +

      + {{ version[1].name }} + {% if version[1].version == stable.version %}(stable){% endif %} + (download) +

      -* [Elixir](https://hexdocs.pm/elixir/master/) - standard library -* [EEx](https://hexdocs.pm/eex/master/) - templating library -* [ExUnit](https://hexdocs.pm/ex_unit/master/) - unit test library -* [IEx](https://hexdocs.pm/iex/master/) - interactive shell -* [Logger](https://hexdocs.pm/logger/master/) - built-in Logger -* [Mix](https://hexdocs.pm/mix/master/) - build tool +{% if version[1].otp_versions %} +Supported Erlang/OTP versions: {% for otp in version[1].otp_versions reversed %}{{ otp }}{% if forloop.last %}{% else %}, {% endif %}{% endfor %}. +{% endif %} -

      v1.3 (download)

      +* [Elixir](https://hexdocs.pm/elixir/{{ version[1].version }}/) - standard library +* [EEx](https://hexdocs.pm/eex/{{ version[1].version }}/) - templating library +* [ExUnit](https://hexdocs.pm/ex_unit/{{ version[1].version }}/) - unit test library +* [IEx](https://hexdocs.pm/iex/{{ version[1].version }}/) - interactive shell +* [Logger](https://hexdocs.pm/logger/{{ version[1].version }}/) - built-in Logger +* [Mix](https://hexdocs.pm/mix/{{ version[1].version }}/) - build tool -* [Elixir](https://hexdocs.pm/elixir/1.3.3/) - standard library -* [EEx](https://hexdocs.pm/eex/1.3.3/) - templating library -* [ExUnit](https://hexdocs.pm/ex_unit/1.3.3/) - unit test library -* [IEx](https://hexdocs.pm/iex/1.3.3/) - interactive shell -* [Logger](https://hexdocs.pm/logger/1.3.3/) - built-in Logger -* [Mix](https://hexdocs.pm/mix/1.3.3/) - build tool +
      +{% endfor %} -

      v1.2 (download)

      +#### Development -* [Elixir](https://hexdocs.pm/elixir/1.2.6/) - standard library -* [EEx](https://hexdocs.pm/eex/1.2.6/) - templating library -* [ExUnit](https://hexdocs.pm/ex_unit/1.2.6/) - unit test library -* [IEx](https://hexdocs.pm/iex/1.2.6/) - interactive shell -* [Logger](https://hexdocs.pm/logger/1.2.6/) - built-in Logger -* [Mix](https://hexdocs.pm/mix/1.2.6/) - build tool - -

      v1.1 (download)

      - -* [Elixir](https://hexdocs.pm/elixir/1.1.1/) - standard library -* [EEx](https://hexdocs.pm/eex/1.1.1/) - templating library -* [ExUnit](https://hexdocs.pm/ex_unit/1.1.1/) - unit test library -* [IEx](https://hexdocs.pm/iex/1.1.1/) - interactive shell -* [Logger](https://hexdocs.pm/logger/1.1.1/) - built-in Logger -* [Mix](https://hexdocs.pm/mix/1.1.1/) - build tool +* [Elixir](https://hexdocs.pm/elixir/main/) - standard library +* [EEx](https://hexdocs.pm/eex/main/) - templating library +* [ExUnit](https://hexdocs.pm/ex_unit/main/) - unit test library +* [IEx](https://hexdocs.pm/iex/main/) - interactive shell +* [Logger](https://hexdocs.pm/logger/main/) - built-in Logger +* [Mix](https://hexdocs.pm/mix/main/) - build tool diff --git a/downloads/cheatsheets/README.md b/downloads/cheatsheets/README.md new file mode 100644 index 000000000..1fb6e673e --- /dev/null +++ b/downloads/cheatsheets/README.md @@ -0,0 +1,21 @@ +This directory contains TeX source files and their corresponding +PDF files, containing small help pages, reminders, compressed +references, also called cheat sheets. + +To produce a .pdf starting from the .tex source, you need `tex`, +and more specifically [LaTeX](https://www.latex-project.org/get/). + +The BasicTex variant can be used as long as the following packages +are available: + + $ tlmgr install courier framed charter enumitem ec helvetica + +To compile a tex file EQUIS.tex into its corresponding EQUIS.pdf: + + $ texi2pdf EQUIS.tex + +You may ignore all output except the very last two lines, which +should look like: + + Output written on EQUIS.pdf (2 pages, 69846 bytes). + Transcript written on EQUIS.log. diff --git a/downloads/cheatsheets/gen-server.pdf b/downloads/cheatsheets/gen-server.pdf new file mode 100644 index 000000000..e67c6125f Binary files /dev/null and b/downloads/cheatsheets/gen-server.pdf differ diff --git a/downloads/cheatsheets/gen-server.tex b/downloads/cheatsheets/gen-server.tex new file mode 100644 index 000000000..5465462d1 --- /dev/null +++ b/downloads/cheatsheets/gen-server.tex @@ -0,0 +1,403 @@ +\documentclass[a4paper]{article} +\usepackage{framed} +\usepackage{tikz} +\usepackage{times,charter} +\usepackage{amssymb} +\usepackage{verbatim} +\usepackage[bindingoffset=0in,% + left=1.3cm,right=1.3cm,top=1.5cm,bottom=0.8cm,% + footskip=0in]{geometry} +\usetikzlibrary{shapes,arrows,calc,backgrounds,shapes.geometric} +% Define a background layer, in which the parchment shape is drawn +\pgfdeclarelayer{background} +\pgfsetlayers{background,main} + +\tikzset{normal border/.style={orange!30!black!10}} + +% Macro to draw the shape behind the text, when it fits +% completely in the page +\def\parchmentframe#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} % Draw the shape behind + \filldraw[normal border,rounded corners=2em,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Macro to draw the shape, when the text will continue in next page +\def\parchmentframetop#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} + \filldraw[normal border,rounded corners,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Macro to draw the shape, when the text continues from previous page +\def\parchmentframebottom#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} + \filldraw[normal border,rounded corners,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Macro to draw the shape, when both the text continues from previous page +% and it will continue in next page +\def\parchmentframemiddle#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} + \filldraw[normal border,rounded corners,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Define the environment which puts the frame. In this case, +% the environment also accepts an argument with an optional +% title (which defaults to ``Example'', which is typeset in a +% box overlaid on the top border +\newenvironment{parchment}[1][Example]{% + \def\FrameCommand{\parchmentframe}% + \def\FirstFrameCommand{\parchmentframetop}% + \def\LastFrameCommand{\parchmentframebottom}% + \def\MidFrameCommand{\parchmentframemiddle}% + \vskip\baselineskip + \MakeFramed {\FrameRestore} + \noindent\tikz\node[rounded corners=2ex, inner sep=2ex, draw=blue!25!yellow, fill=white, dashed, anchor=west, overlay] at (0em, 2em) {\sffamily#1};\par}% +{\endMakeFramed} + +\setlength\parindent{0pt} + +% Main document, example of usage +\pagestyle{empty} + +\usepackage{enumitem} +\begin{document} +\fontsize{9.5}{11}\selectfont +\setlist[itemize]{leftmargin=0em} + +% We need layers to draw the block diagram +\pgfdeclarelayer{background} +\pgfdeclarelayer{foreground} +\pgfsetlayers{background,main,foreground} + +% Define a few styles and constants +\tikzstyle{left-title}=[draw, fill=blue!20, minimum width=1.6em, minimum height=5em,anchor=north east] +\tikzstyle{top-title}=[draw, fill=blue!20, minimum width=5em, minimum height=1.6em] +\tikzstyle{main-block} = [left-title, text width=30em, inner sep=8pt, fill=red!15!yellow!40, minimum height=5em, rounded corners] +\tikzstyle{immediate} = [left-title, text width=17em, inner sep=8pt, fill=red!15!yellow!40, minimum height=5em, rounded corners] +\tikzstyle{explain} = [left-title, text width=17em, inner sep=8pt, fill=black!5, minimum height=5em, rounded corners,anchor=south west] + +\begin{comment} +\begin{parchment}[commentarii de bello gallico] + Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur. Hi omnes lingua, institutis, legibus inter se differunt. Gallos ab Aquitanis Garumna flumen, a Belgis Matrona et Sequana dividit. Horum omnium fortissimi sunt Belgae, propterea quod a cultu atque humanitate provinciae longissime absunt, minimeque ad eos mercatores saepe commeant atque ea quae ad effeminandos animos pertinent important, proximique sunt Germanis, qui trans Rhenum incolunt, quibuscum continenter bellum gerunt. Qua de causa Helvetii quoque reliquos Gallos virtute praecedunt, quod fere cotidianis proeliis cum Germanis contendunt, cum aut suis finibus eos prohibent aut ipsi in eorum finibus bellum gerunt. Eorum una pars, quam Gallos obtinere dictum est, initium capit a flumine Rhodano, continetur Garumna flumine, Oceano, finibus Belgarum, attingit etiam ab Sequanis et Helvetiis flumen Rhenum, vergit ad septentriones. Belgae ab extremis Galliae finibus oriuntur, pertinent ad inferiorem partem fluminis Rheni, spectant in septentrionem et orientem solem. Aquitania a Garumna flumine ad Pyrenaeos montes et eam partem Oceani quae est ad Hispaniam pertinet; spectat inter occasum solis et septentriones. +\end{parchment} +\end{comment} +\section*{GenServer - a cheat sheet} +\begin{itemize} + \setlength\itemsep{0em} +\item[---] last version: \verb|https://elixir-lang.org/downloads/cheatsheets/gen-server.pdf| +\item[---] reference: \verb|https://hexdocs.pm/elixir/GenServer.html| +\end{itemize} +\begin{comment} + copyright 2019 by Mario Frasca et al. + + based on a work by Benjamin Tan Wei Hao, + rewritten as a tex/tikz document by Mario Frasca, + with José Valim validating the content. +\end{comment} + +\begin{parchment}[initialization: .start $\rightarrow$ \bf\texttt{init/1}] + \begin{tikzpicture} + \node (client) [main-block] {% + \vspace*{-\baselineskip} +\begin{verbatim} +def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, match_this, opts) +end +\end{verbatim} + }; + \path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + + \path (client.north east)+(3em,0) node (immediate) [immediate,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:ok, pid} +:ignore +{:error, term} +\end{verbatim} +}; + \path (immediate.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + + \path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def init(match_this) do + # process input and compute result + result +end +\end{verbatim} +}; + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:ok, state} +{:ok, state, then_what} + +{:stop, reason} +:ignore +\end{verbatim} +}; +\path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result = }}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\path (result.south east)+(3em,0) node (reason) [immediate,anchor=south west,fill=green!30!yellow!18!white] { +One of \verb|:normal|, \verb|:shutdown|, \verb|{:shutdown, _}|, or any other value. See the footnote for a link to the complete reference. +}; +\path (reason.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}reason = }}; +\path (reason.north west)+(0,2pt) node [top-title,anchor=south west,fill=green!25] {applies globally}; + + \draw[->,draw=blue!50!black,dashed,thick,anchor=north west] (-7.3em,-3.3em) .. controls (-7em,-7em) and (-22.45em,-1em) .. (-22.45em,-5.5em) -- (-22.45em,-6.9em); + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[termination: .stop $\rightarrow$ \bf\texttt{terminate/2}] +\begin{tikzpicture} + \node (client) [main-block] {% + \vspace*{-\baselineskip} +\begin{verbatim} +def stop(pid, reason \\ :normal, + timeout \\ :infinity) do + GenServer.stop(pid, reason, timeout) +end +\end{verbatim} +}; +\path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + +\path (client.north east)+(3em,0) node (immediate) [immediate,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +:ok +\end{verbatim} +}; +\path (immediate.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + +\path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def terminate(reason, state) do + # perform cleanup + # result will not be used +end +\end{verbatim} +}; + +\path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + +\path (callback.south east)+(3em,0) node (then-what) [explain] { + \verb|terminate/2| is also called when \verb|:stop| is returned and in case of errors, when \verb|Process.flag(:trap_exit)| is true. +}; + \path (then-what.north west)+(-2pt,0) node [left-title,fill=white] {\rotatebox{90}{:stop}}; + + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[asynchronous operation: .cast $\rightarrow$ \bf\texttt{handle\_cast/2}] +\begin{tikzpicture} + \node (client) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def your_api_async_op(pid, args) do + GenServer.cast(pid, match_this) +end +\end{verbatim} +}; + \path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + +\path (client.north east)+(3em,0) node (result) [immediate,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +:ok +\end{verbatim} +}; + \path (result.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + + \path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_cast(match_this, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \draw[->,draw=blue!50!black,dashed,thick] (-15.1em,-3.3em) .. controls (-15.1em,-5.4em) and (-18.2em,-3em) .. (-18.2em,-5.8em) -- (-18.2em,-6.8em); + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, state} +\end{verbatim} +}; + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + + \path (result.south east)+(3em,0) node (then-what) [immediate,anchor=south west,fill=green!30!yellow!18!white] { + \vspace*{-\baselineskip} +\begin{verbatim} +timeout_milliseconds +:hibernate +{:continue, match_this} +\end{verbatim} +}; +\path (then-what.north west)+(0,2pt) node [top-title,anchor=south west,fill=green!25] {applies globally}; + \path (then-what.south west)+(-2pt,0) node [left-title,anchor=south east,fill=green!25] {\rotatebox{90}{\^{}then\_what =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[synchronous operation: .call $\rightarrow$ \bf\texttt{handle\_call/3}] +\begin{tikzpicture} + \node (client) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def your_api_sync_op(pid, args) do + GenServer.call(pid, match_this) +end +\end{verbatim} +}; + \path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + + \path (client.north east)+(3em,0) node (immediate) [immediate,anchor=north west] {waits for callback, receives \verb|reply| if result matches \verb|{:reply, reply, ...}| or \verb|{:stop, _, reply, _}|.}; + \path (immediate.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + + \path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_call(match_this, from, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \draw[->,draw=blue!50!black,dashed,thick] (-15.1em,-3.3em) .. controls (-15.1em,-5.4em) and (-18.2em,-3em) .. (-18.2em,-5.8em) -- (-18.2em,-6.8em); + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:reply, reply, state} +{:reply, reply, state, then_what} +\end{verbatim} + +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, reply, state} +\end{verbatim} +}; + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + +\path (result.south east)+(3em,0) node (reply) [immediate,anchor=south west] {user defined}; + \path (reply.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}reply =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-72pt) -- ($ (result.north east)+(0,-72pt) $); + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} + +\end{parchment} + +\begin{parchment}[handling messages: $\rightarrow$ \bf\texttt{handle\_info/2}] +\begin{tikzpicture} + \node (callback) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_info(match_this, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, state} +\end{verbatim} +}; + + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[\bf\texttt{\^{}then\_what = \{:continue, match\_this\}} $\rightarrow$ \bf\texttt{handle\_continue/2}] +\begin{tikzpicture} + \node (callback) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_continue(match_this, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, state} +\end{verbatim} +}; + + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} +\end{parchment} +\begin{parchment}[footnotes] + \vspace*{-\baselineskip} + \begin{itemize} + \setlength\itemsep{0em} + \item[---] More on exit reasons: \verb|https://hexdocs.pm/elixir/Supervisor.html#module-exit-reasons-and-restarts| + \item[---] use \verb|@impl true| before each definition to guarantee it matches the equivalent \verb|GenServer| callback. + \item[---] callbacks not listed here are: \verb|code_change/3| and \verb|format_status/2|. + \item[---] source: \verb|https://github.com/elixir-lang/elixir-lang.github.com| + \item[---] copyright: by its authors, listed in the source --- license: CC:BY-SA% + \end{itemize} +\end{parchment} + +\end{document} diff --git a/downloads/logos/elixir-drop-only.png b/downloads/logos/elixir-drop-only.png new file mode 100644 index 000000000..97008feb9 Binary files /dev/null and b/downloads/logos/elixir-drop-only.png differ diff --git a/downloads/logos/elixir-horizontal.png b/downloads/logos/elixir-horizontal.png new file mode 100644 index 000000000..def595473 Binary files /dev/null and b/downloads/logos/elixir-horizontal.png differ diff --git a/downloads/logos/elixir-vertical.png b/downloads/logos/elixir-vertical.png new file mode 100644 index 000000000..52391a543 Binary files /dev/null and b/downloads/logos/elixir-vertical.png differ diff --git a/elixir.csv b/elixir.csv index cc86461db..48961d2c7 100644 --- a/elixir.csv +++ b/elixir.csv @@ -1,22 +1,25 @@ -version,url_precompiled,release_type,windows_installer_compat,sha1,sha512 -1.4.0-rc.0,https://github.com/elixir-lang/elixir/releases/download/v1.4.0-rc.0/Precompiled.zip,prerelease,1,23257bcd484e767fa1149cf8046da1f865a6ad3b,9756080922820dad162c08d794eeec1f6655806bdaddeabf4dfb0ae5128363d6 -1.3.4,https://github.com/elixir-lang/elixir/releases/download/v1.3.4/Precompiled.zip,release,1,, -1.3.3,https://github.com/elixir-lang/elixir/releases/download/v1.3.3/Precompiled.zip,release,1,, -1.3.2,https://github.com/elixir-lang/elixir/releases/download/v1.3.2/Precompiled.zip,release,1,, -1.3.1,https://github.com/elixir-lang/elixir/releases/download/v1.3.1/Precompiled.zip,release,1,, -1.3.0,https://github.com/elixir-lang/elixir/releases/download/v1.3.0/Precompiled.zip,release,1,, -1.2.6,https://github.com/elixir-lang/elixir/releases/download/v1.2.6/Precompiled.zip,release,1,, -1.2.5,https://github.com/elixir-lang/elixir/releases/download/v1.2.5/Precompiled.zip,release,1,, -1.2.4,https://github.com/elixir-lang/elixir/releases/download/v1.2.4/Precompiled.zip,release,1,, -1.2.3,https://github.com/elixir-lang/elixir/releases/download/v1.2.3/Precompiled.zip,release,1,, -1.2.2,https://github.com/elixir-lang/elixir/releases/download/v1.2.2/Precompiled.zip,release,1,, -1.2.1,https://github.com/elixir-lang/elixir/releases/download/v1.2.1/Precompiled.zip,release,1,, -1.2.0,https://github.com/elixir-lang/elixir/releases/download/v1.2.0/Precompiled.zip,release,1,, -1.1.1,https://github.com/elixir-lang/elixir/releases/download/v1.1.1/Precompiled.zip,release,1,, -1.1.0,https://github.com/elixir-lang/elixir/releases/download/v1.1.0/Precompiled.zip,release,1,, -1.0.5,https://github.com/elixir-lang/elixir/releases/download/v1.0.5/Precompiled.zip,release,1,, -1.0.4,https://github.com/elixir-lang/elixir/releases/download/v1.0.4/Precompiled.zip,release,1,, -1.0.3,https://github.com/elixir-lang/elixir/releases/download/v1.0.3/Precompiled.zip,release,1,, -1.0.2,https://github.com/elixir-lang/elixir/releases/download/v1.0.2/Precompiled.zip,release,1,, -1.0.1,https://github.com/elixir-lang/elixir/releases/download/v1.0.1/Precompiled.zip,release,1,, -1.0.0,https://github.com/elixir-lang/elixir/releases/download/v1.0.0/Precompiled.zip,release,1,, +version,url_precompiled,release_type,windows_installer_compat,sha1,sha512,min_otp_release +1.14.5,https://github.com/elixir-lang/elixir/releases/download/v1.14.5/elixir-otp-25.zip,release,1,6fc78c40bd13afbd8531dff6a08846d095c1d4ea,f3b35d9fa61da7e93c9979cb8a08f64a9ce7074aeda66fae994f2a4ea2e4f82e,25.0 +1.14.4,https://github.com/elixir-lang/elixir/releases/download/v1.14.4/elixir-otp-25.zip,release,1,2c52e5a2357030ecb31bbf3dff5835122b1658ab,a5b7aadfd896e691a6494f9079fcd6f1209adcbd93f2d40e5770d80edc0f33e6,25.0 +1.14.2,https://github.com/elixir-lang/elixir/releases/download/v1.14.2/elixir-otp-25.zip,release,1,4bdbb762f102e6318388389a4b44e3a132534ae3,2ab159e875a8d407dde10ee279446d469022acf8128e4f390556ab44e8918a06,25.0 +1.14.1,https://github.com/elixir-lang/elixir/releases/download/v1.14.1/elixir-otp-25.zip,release,1,199e252f71ee82cd2a0026e496f230e9ccd8c1eb,40a880e817bc188469e0adf73f41acb82b6a5b9e5da5cbcb55d89d98dcafe5e5,25.0 +1.14.0,https://github.com/elixir-lang/elixir/releases/download/v1.14.0/elixir-otp-25.zip,release,1,c7ff5116fed9ff7b34f07d3187a8c7c590a79626,55d5708605aa0c81c06acc284e1d40a2853942988f3a6c399759d8ad065e46d0,25.0 +1.13.4,https://github.com/elixir-lang/elixir/releases/download/v1.13.4/Precompiled.zip,release,1,325fbdde4f0a5701bb8b9d455175b85ff41470d7,e64c714e80cd9657b8897d725f6d78f251d443082f6af5070caec863c18068c97af6bdda156c3b3390e0a2b84f77c2ad3378a42913f64bb583fb5251fa49e619,22.0 +1.13.3,https://github.com/elixir-lang/elixir/releases/download/v1.13.3/Precompiled.zip,release,1,7a2d0ff13beadcba3f566d692d960dcd785df5c8,93132c03a16479cfd48c509e2c5ee145b9062d77d528ac2eaeae460f4349f138286f14d34a1ee884e6c76081fe1bf52d27788b944ef06feaa40c07bec41a0a27,22.0 +1.13.2,https://github.com/elixir-lang/elixir/releases/download/v1.13.2/Precompiled.zip,release,1,53bf917f18fc210dcf252e346453fd8c04f2ac6b,74cc0b3d7ddb0156d6695b3b08319d0c01fcfac053407b0bf8b456013d21d1b37ffd5cba389557edfb6af329fbed07c8cd1061a6698b60e1b16a70c898720ec2,22.0 +1.13.1,https://github.com/elixir-lang/elixir/releases/download/v1.13.1/Precompiled.zip,release,1,40762ffbef86cbbfd0c79a94e057fb987dc882ff,bb02ead0d4ccf499ff0473fbbb17fd12de4ba476b463c0452138bc1ef9004547166e75ffe7c6f96b9497adc9fecbda4ad57bfe0f17f1ba95d8339a0e98c29b03,22.0 +1.13.0,https://github.com/elixir-lang/elixir/releases/download/v1.13.0/Precompiled.zip,release,1,0f196f4de406882b5a3bcee7ac9e9b0c61ed1459,58ffe87d6eb89435d8605aee04556aa1a1ba25cf2fa7688d0a5da162d7d7c57b47b2f726b365a7aeb18832bf08de3db5d3ec0ed45e13da276438679f29e5e3ac,22.0 +1.12.3,https://github.com/elixir-lang/elixir/releases/download/v1.12.3/Precompiled.zip,release,1,db3c8a5470dee211168707c432a1f8002960e966,db092caa32b55195eeb24a17e0ab98bb2fea38d2f638bc42fee45a6dfcd3ba0782618d27e281c545651f93914481866b9d34b6d284c7f763d197e87847fdaef4,22.0 +1.12.2,https://github.com/elixir-lang/elixir/releases/download/v1.12.2/Precompiled.zip,release,1,051c2728bfd679d1d6f74d11f25ca679491cd6b4,38eb2281032b0cb096ef5e61f048c5374d6fb9bf4078ab8f9526a42e16e7c661732a632b55d6072328eedf87a47e6eeb3f0e3f90bba1086239c71350f90c75e5,22.0 +1.12.1,https://github.com/elixir-lang/elixir/releases/download/v1.12.1/Precompiled.zip,release,1,4a1269b95c719e0295cd6b57a1c447e1b91563ef,66e877486606f232f36cbe76cdd63bc4ee5e75e27d6292bb602ab2c88b4dd7da6684e6ff9cdb2020e7f916ee2d58016e2d1e2cbd5e579f297ab8eae660cb5dcb,22.0 +1.12.0,https://github.com/elixir-lang/elixir/releases/download/v1.12.0/Precompiled.zip,release,1,b607b2d0e78e1ae6f9d112ba3e1b617a1ee7580b,57e970640f7e273a62fbde60545ed85c18d667e194d0d3483b941ea3164f12664cc0c186968c1f341253f1907157ae88953b5caa413204f93bba08710fdc0b50,22.0 +1.11.4,https://github.com/elixir-lang/elixir/releases/download/v1.11.4/Precompiled.zip,release,1,2b421e7f4b8d62af5c9238f5cf127938d3523f23,4d8ead533a7bd35b41669be0d4548b612d5cc17723da67cfdf996ab36522fd0163215915a970675c6ebcba4dbfc7a46e644cb144b16087bc9417b385955a1e79,21.0 +1.11.3,https://github.com/elixir-lang/elixir/releases/download/v1.11.3/Precompiled.zip,release,1,430735e03d8dfcba3324010d397ab7871b30d87d,16cdd30e41737ba9d659e506b4aa79465ede2d93d27a720d6a3649b8100da9f7dee154a913438724651a83aea77ddef33ac1489bb7ae283b39fbfe20dcd77db4,21.0 +1.11.2,https://github.com/elixir-lang/elixir/releases/download/v1.11.2/Precompiled.zip,release,1,1c54c01b5449d631e673e4aaf205522eaeb8dbdc,7b6ece70d2db6cf4b02c3e72f0ae2edc68e13a2651876525eab08e3c3c3f1f88d294e3f412c3fb736bc84ba4c2897930101b0ac715e322f1554f29c69e2d1e77,21.0 +1.11.1,https://github.com/elixir-lang/elixir/releases/download/v1.11.1/Precompiled.zip,release,1,f03301dd10072f3643a4c1e5f012a49bfc493608,c70e7d3e338ab2449301c7864a2dd96e8d474f3fee533a66706c67cd733db74c2d529b9887961db9f07530eeed4cce917c480e5e8c4201060c7fa494e8b376da,21.0 +1.11.0,https://github.com/elixir-lang/elixir/releases/download/v1.11.0/Precompiled.zip,release,1,4654fed3bbd3e4ebc0c15cfaec003e84080eabd3,6293f2a0ca87872789a9b3909e782cde77409e8b3f1699f42c95ef55360bdba27c69fe6aaa1c843ef004cc4ded9d92e4cc0053f845c0208ec38f1d11945cae07,21.0 +1.10.4,https://github.com/elixir-lang/elixir/releases/download/v1.10.4/Precompiled.zip,release,1,5b400c829c4f239ac89a7eb97aac642b961f38fd,9727ae96d187d8b64e471ff0bb5694fcd1009cdcfd8b91a6b78b7542bb71fca59869d8440bb66a2523a6fec025f1d23394e7578674b942274c52b44e19ba2d43,21.0 +1.10.3,https://github.com/elixir-lang/elixir/releases/download/v1.10.3/Precompiled.zip,release,1,382fa22e6f184c0cc87fcfbfa0053c349d85f6f1,fc6d06ad4cc596b2b6e4f01712f718200c69f3b9c49c7d3b787f9a67b36482658490cf01109b0b0842fc9d88a27f64a9aba817231498d99fa01fa99688263d55,21.0 +1.10.2,https://github.com/elixir-lang/elixir/releases/download/v1.10.2/Precompiled.zip,release,1,830264881231e711d8bade711ed2819a2d10b918,532f43f08a29b5cbfca5a544c337b4a6d336d372ba0a5b3393681426d57ecaa85960e8eb3548aea26f213cc36914c6b66b83707cd719e27dc34c40efadb9f0b9,21.0 +1.10.1,https://github.com/elixir-lang/elixir/releases/download/v1.10.1/Precompiled.zip,release,1,606ffaff6c7c3d278d7daa1fee653aa8183f225f,e8809aff909ca6e2271493690a6a095959e551f6d04e2d384ead0d6cfde23a3707492b53563b9a78080509b44728a63645dc37be108cd6ea3f9501f5d616fe1d,21.0 +1.10.0,https://github.com/elixir-lang/elixir/releases/download/v1.10.0/Precompiled.zip,release,1,0c3f2206131f6d50f9b4e3264dcb8757c52294e4,1e9286391281cd53e5cc5452cdf9ee586c50648800701a1ab80ef7a5a4ef4052f75149235ba348e560d8d5247a0f476c27f481f53e05cbe29244d0b1a25d6586,21.0 diff --git a/erlang.csv b/erlang.csv index ae3353cbc..7f0e37d55 100644 --- a/erlang.csv +++ b/erlang.csv @@ -1,6 +1,5 @@ version_otp,version_erts,url_win32,url_win64 -19.0,8.0,http://erlang.org/download/otp_win32_19.0.exe,http://erlang.org/download/otp_win64_19.0.exe -18.3,7.3,http://erlang.org/download/otp_win32_18.3.exe,http://erlang.org/download/otp_win64_18.3.exe -18.1,7.1,http://erlang.org/download/otp_win32_18.1.exe,http://erlang.org/download/otp_win64_18.1.exe -17.5,6.4,http://erlang.org/download/otp_win32_17.5.exe,http://erlang.org/download/otp_win64_17.5.exe -17.4,6.3,http://erlang.org/download/otp_win32_17.4.exe,http://erlang.org/download/otp_win64_17.4.exe +25.3,13.3,https://github.com/erlang/otp/releases/download/OTP-25.3.1/otp_win32_25.3.1.exe,https://github.com/erlang/otp/releases/download/OTP-25.3.1/otp_win64_25.3.1.exe +24.3,12.3,https://github.com/erlang/otp/releases/download/OTP-24.3.4.2/otp_win32_24.3.4.2.exe,https://github.com/erlang/otp/releases/download/OTP-24.3.4.2/otp_win64_24.3.4.2.exe +23.3,11.2,https://github.com/erlang/otp/releases/download/OTP-23.3.4.8/otp_win32_23.3.4.8.exe,https://github.com/erlang/otp/releases/download/OTP-23.3.4.8/otp_win64_23.3.4.8.exe +22.3,10.7,http://www.erlang.org/download/otp_win32_22.3.exe,http://www.erlang.org/download/otp_win64_22.3.exe diff --git a/getting-started/alias-require-and-import.markdown b/getting-started/alias-require-and-import.markdown index 33e07a16e..eb771d715 100644 --- a/getting-started/alias-require-and-import.markdown +++ b/getting-started/alias-require-and-import.markdown @@ -1,228 +1,5 @@ --- -layout: getting-started -title: alias, require and import +layout: redirect +sitemap: false +redirect_to: alias-require-and-import --- - -# {{ page.title }} - -{% include toc.html %} - -In order to facilitate software reuse, Elixir provides three directives (`alias`, `require` and `import`) plus a macro called `use` summarized below: - -```elixir -# Alias the module so it can be called as Bar instead of Foo.Bar -alias Foo.Bar, as: Bar - -# Ensure the module is compiled and available (usually for macros) -require Foo - -# Import functions from Foo so they can be called without the `Foo.` prefix -import Foo - -# Invokes the custom code defined in Foo as an extension point -use Foo -``` - -We are going to explore them in detail now. Keep in mind the first three are called directives because they have **lexical scope**, while `use` is a common extension point. - -## alias - -`alias` allows you to set up aliases for any given module name. Imagine our `Math` module uses a special list implementation for doing math specific operations: - -```elixir -defmodule Math do - alias Math.List, as: List -end -``` - -From now on, any reference to `List` will automatically expand to `Math.List`. In case one wants to access the original `List`, it can be done by prefixing the module name with `Elixir.`: - -```elixir -List.flatten #=> uses Math.List.flatten -Elixir.List.flatten #=> uses List.flatten -Elixir.Math.List.flatten #=> uses Math.List.flatten -``` - -> Note: All modules defined in Elixir are defined inside a main Elixir namespace. However, for convenience, you can omit "Elixir." when referencing them. - -Aliases are frequently used to define shortcuts. In fact, calling `alias` without an `:as` option sets the alias automatically to the last part of the module name, for example: - -```elixir -alias Math.List -``` - -Is the same as: - -```elixir -alias Math.List, as: List -``` - -Note that `alias` is **lexically scoped**, which allows you to set aliases inside specific functions: - -```elixir -defmodule Math do - def plus(a, b) do - alias Math.List - # ... - end - - def minus(a, b) do - # ... - end -end -``` - -In the example above, since we are invoking `alias` inside the function `plus/2`, the alias will be valid only inside the function `plus/2`. `minus/2` won't be affected at all. - -## require - -Elixir provides macros as a mechanism for meta-programming (writing code that generates code). - -Macros are chunks of code that are executed and expanded at compilation time. This means, in order to use a macro, we need to guarantee its module and implementation are available during compilation. This is done with the `require` directive: - -```iex -iex> Integer.is_odd(3) -** (CompileError) iex:1: you must require Integer before invoking the macro Integer.is_odd/1 -iex> require Integer -Integer -iex> Integer.is_odd(3) -true -``` - -In Elixir, `Integer.is_odd/1` is defined as a macro so that it can be used as a guard. This means that, in order to invoke `Integer.is_odd/1`, we need to first require the `Integer` module. - -In general a module does not need to be required before usage, except if we want to use the macros available in that module. An attempt to call a macro that was not loaded will raise an error. Note that like the `alias` directive, `require` is also lexically scoped. We will talk more about macros in a later chapter. - -## import - -We use `import` whenever we want to easily access functions or macros from other modules without using the fully-qualified name. For instance, if we want to use the `duplicate/2` function from the `List` module several times, we can import it: - -```iex -iex> import List, only: [duplicate: 2] -List -iex> duplicate :ok, 3 -[:ok, :ok, :ok] -``` - -In this case, we are importing only the function `duplicate` (with arity 2) from `List`. Although `:only` is optional, its usage is recommended in order to avoid importing all the functions of a given module inside the namespace. `:except` could also be given as an option in order to import everything in a module *except* a list of functions. - -`import` also supports `:macros` and `:functions` to be given to `:only`. For example, to import all macros, one could write: - -```elixir -import Integer, only: :macros -``` - -Or to import all functions, you could write: - -```elixir -import Integer, only: :functions -``` - -Note that `import` is **lexically scoped** too. This means that we can import specific macros or functions inside function definitions: - -```elixir -defmodule Math do - def some_function do - import List, only: [duplicate: 2] - duplicate(:ok, 10) - end -end -``` - -In the example above, the imported `List.duplicate/2` is only visible within that specific function. `duplicate/2` won't be available in any other function in that module (or any other module for that matter). - -Note that `import`ing a module automatically `require`s it. - -## use - -Although not a directive, `use` is a macro tightly related to `require` that allows you to use a module in the current context. The `use` macro is frequently used by developers to bring external functionality into the current lexical scope, often modules. - -For example, in order to write tests using the ExUnit framework, a developer should use the `ExUnit.Case` module: - -```elixir -defmodule AssertionTest do - use ExUnit.Case, async: true - - test "always pass" do - assert true - end -end -``` - -Behind the scenes, `use` requires the given module and then calls the `__using__/1` callback on it allowing the module to inject some code into the current context. Generally speaking, the following module: - -```elixir -defmodule Example do - use Feature, option: :value -end -``` - -is compiled into - -```elixir -defmodule Example do - require Feature - Feature.__using__(option: :value) -end -``` - -With this we have almost finished our tour of Elixir modules. The last topic to cover is module attributes. - -## Understanding Aliases - -At this point, you may be wondering: what exactly is an Elixir alias and how is it represented? - -An alias in Elixir is a capitalized identifier (like `String`, `Keyword`, etc) which is converted to an atom during compilation. For instance, the `String` alias translates by default to the atom `:"Elixir.String"`: - -```iex -iex> is_atom(String) -true -iex> to_string(String) -"Elixir.String" -iex> :"Elixir.String" == String -true -``` - -By using the `alias/2` directive, we are changing the atom the alias expands to. - -Aliases expand to atoms because in the Erlang VM (and consequently Elixir) modules are always represented by atoms. For example, that's the mechanism we use to call Erlang modules: - -```iex -iex> :lists.flatten([1, [2], 3]) -[1, 2, 3] -``` - -## Module nesting - -Now that we have talked about aliases, we can talk about nesting and how it works in Elixir. Consider the following example: - -```elixir -defmodule Foo do - defmodule Bar do - end -end -``` - -The example above will define two modules: `Foo` and `Foo.Bar`. The second can be accessed as `Bar` inside `Foo` as long as they are in the same lexical scope. The code above is exactly the same as: - -```elixir -defmodule Elixir.Foo do - defmodule Elixir.Foo.Bar do - end - alias Elixir.Foo.Bar, as: Bar -end -``` - -If, later, the `Bar` module is moved outside the `Foo` module definition, it must be referenced by its full name (`Foo.Bar`) or an alias must be set using the `alias` directive discussed above. - -**Note**: in Elixir, you don't have to define the `Foo` module before being able to define the `Foo.Bar` module, as the language translates all module names to atoms. You can define arbitrarily-nested modules without defining any module in the chain (e.g., `Foo.Bar.Baz` without defining `Foo` or `Foo.Bar` first). - -As we will see in later chapters, aliases also play a crucial role in macros, to guarantee they are hygienic. - -## Multi alias/import/require/use - -From Elixir v1.2, it is possible to alias, import or require multiple modules at once. This is particularly useful once we start nesting modules, which is very common when building Elixir applications. For example, imagine you have an application where all modules are nested under `MyApp`, you can alias the modules `MyApp.Foo`, `MyApp.Bar` and `MyApp.Baz` at once as follows: - -```elixir -alias MyApp.{Foo, Bar, Baz} -``` diff --git a/getting-started/basic-operators.markdown b/getting-started/basic-operators.markdown index a29da40b6..459bdf53c 100644 --- a/getting-started/basic-operators.markdown +++ b/getting-started/basic-operators.markdown @@ -1,116 +1,5 @@ --- -layout: getting-started -title: Basic operators +layout: redirect +sitemap: false +redirect_to: lists-and-tuples --- - -# {{ page.title }} - -{% include toc.html %} - -In the [previous chapter](/getting-started/basic-types.html), we saw Elixir provides `+`, `-`, `*`, `/` as arithmetic operators, plus the functions `div/2` and `rem/2` for integer division and remainder. - -Elixir also provides `++` and `--` to manipulate lists: - -```iex -iex> [1, 2, 3] ++ [4, 5, 6] -[1, 2, 3, 4, 5, 6] -iex> [1, 2, 3] -- [2] -[1, 3] -``` - -String concatenation is done with `<>`: - -```iex -iex> "foo" <> "bar" -"foobar" -``` - -Elixir also provides three boolean operators: `or`, `and` and `not`. These operators are strict in the sense that they expect a boolean (`true` or `false`) as their first argument: - -```iex -iex> true and true -true -iex> false or is_atom(:example) -true -``` - -Providing a non-boolean will raise an exception: - -```iex -iex> 1 and true -** (ArgumentError) argument error: 1 -``` - -`or` and `and` are short-circuit operators. They only execute the right side if the left side is not enough to determine the result: - -```iex -iex> false and raise("This error will never be raised") -false -iex> true or raise("This error will never be raised") -true -``` - -> Note: If you are an Erlang developer, `and` and `or` in Elixir actually map to the `andalso` and `orelse` operators in Erlang. - -Besides these boolean operators, Elixir also provides `||`, `&&` and `!` which accept arguments of any type. For these operators, all values except `false` and `nil` will evaluate to true: - -```iex -# or -iex> 1 || true -1 -iex> false || 11 -11 - -# and -iex> nil && 13 -nil -iex> true && 17 -17 - -# ! -iex> !true -false -iex> !1 -false -iex> !nil -true -``` - -As a rule of thumb, use `and`, `or` and `not` when you are expecting booleans. If any of the arguments are non-boolean, use `&&`, `||` and `!`. - -Elixir also provides `==`, `!=`, `===`, `!==`, `<=`, `>=`, `<`, and `>` as comparison operators: - -```iex -iex> 1 == 1 -true -iex> 1 != 2 -true -iex> 1 < 2 -true -``` - -The difference between `==` and `===` is that the latter is more strict when comparing integers and floats: - -```iex -iex> 1 == 1.0 -true -iex> 1 === 1.0 -false -``` - -In Elixir, we can compare two different data types: - -```iex -iex> 1 < :atom -true -``` - -The reason we can compare different data types is pragmatism. Sorting algorithms don't need to worry about different data types in order to sort. The overall sorting order is defined below: - - number < atom < reference < function < port < pid < tuple < map < list < bitstring - -You don't actually need to memorize this ordering, it's enough to know that this ordering exists. - -For reference information about operators (and ordering), check the [reference page on operators](/docs/master/elixir/operators.html). - -In the next chapter, we are going to discuss some basic functions, data type conversions, and a bit of control flow. diff --git a/getting-started/basic-types.markdown b/getting-started/basic-types.markdown index 31daa367d..2ab92484a 100644 --- a/getting-started/basic-types.markdown +++ b/getting-started/basic-types.markdown @@ -1,388 +1,5 @@ --- -layout: getting-started -title: Basic types +layout: redirect +sitemap: false +redirect_to: basic-types --- - -# {{ page.title }} - -{% include toc.html %} - -In this chapter we will learn more about Elixir basic types: integers, floats, booleans, atoms, strings, lists and tuples. Some basic types are: - -```iex -iex> 1 # integer -iex> 0x1F # integer -iex> 1.0 # float -iex> true # boolean -iex> :atom # atom / symbol -iex> "elixir" # string -iex> [1, 2, 3] # list -iex> {1, 2, 3} # tuple -``` - -## Basic arithmetic - -Open up `iex` and type the following expressions: - -```iex -iex> 1 + 2 -3 -iex> 5 * 5 -25 -iex> 10 / 2 -5.0 -``` - -Notice that `10 / 2` returned a float `5.0` instead of an integer `5`. This is expected. In Elixir, the operator `/` always returns a float. If you want to do integer division or get the division remainder, you can invoke the `div` and `rem` functions: - -```iex -iex> div(10, 2) -5 -iex> div 10, 2 -5 -iex> rem 10, 3 -1 -``` - -Notice that parentheses are not required in order to invoke a function. - -Elixir also supports shortcut notations for entering binary, octal, and hexadecimal numbers: - -```iex -iex> 0b1010 -10 -iex> 0o777 -511 -iex> 0x1F -31 -``` - -Float numbers require a dot followed by at least one digit and also support `e` for the exponent number: - -```iex -iex> 1.0 -1.0 -iex> 1.0e-10 -1.0e-10 -``` - -Floats in Elixir are 64 bit double precision. - -You can invoke the `round` function to get the closest integer to a given float, or the `trunc` function to get the integer part of a float. - -```iex -iex> round(3.58) -4 -iex> trunc(3.58) -3 -``` - -## Booleans - -Elixir supports `true` and `false` as booleans: - -```iex -iex> true -true -iex> true == false -false -``` - -Elixir provides a bunch of predicate functions to check for a value type. For example, the `is_boolean/1` function can be used to check if a value is a boolean or not: - -> Note: Functions in Elixir are identified by name and by number of arguments (i.e. arity). Therefore, `is_boolean/1` identifies a function named `is_boolean` that takes 1 argument. `is_boolean/2` identifies a different (nonexistent) function with the same name but different arity. - -```iex -iex> is_boolean(true) -true -iex> is_boolean(1) -false -``` - -You can also use `is_integer/1`, `is_float/1` or `is_number/1` to check, respectively, if an argument is an integer, a float, or either. - -> Note: At any moment you can type `h` in the shell to print information on how to use the shell. The `h` helper can also be used to access documentation for any function. For example, typing `h is_integer/1` is going to print the documentation for the `is_integer/1` function. It also works with operators and other constructs (try `h ==/2`). - -## Atoms - -Atoms are constants where their name is their own value. Some other languages call these symbols: - -```iex -iex> :hello -:hello -iex> :hello == :world -false -``` - -The booleans `true` and `false` are, in fact, atoms: - -```iex -iex> true == :true -true -iex> is_atom(false) -true -iex> is_boolean(:false) -true -``` - -## Strings - -Strings in Elixir are inserted between double quotes, and they are encoded in UTF-8: - -```iex -iex> "hellö" -"hellö" -``` - -> Note: if you are running on Windows, there is a chance your terminal does not use UTF-8 by default. You can change the encoding of your current session by running `chcp 65001` before entering IEx. - -Elixir also supports string interpolation: - -```iex -iex> "hellö #{:world}" -"hellö world" -``` - -Strings can have line breaks in them. You can introduce them using escape sequences: - -```iex -iex> "hello -...> world" -"hello\nworld" -iex> "hello\nworld" -"hello\nworld" -``` - -You can print a string using the `IO.puts/1` function from the `IO` module: - -```iex -iex> IO.puts "hello\nworld" -hello -world -:ok -``` - -Notice the `IO.puts/1` function returns the atom `:ok` as result after printing. - -Strings in Elixir are represented internally by binaries which are sequences of bytes: - -```iex -iex> is_binary("hellö") -true -``` - -We can also get the number of bytes in a string: - -```iex -iex> byte_size("hellö") -6 -``` - -Notice the number of bytes in that string is 6, even though it has 5 characters. That's because the character "ö" takes 2 bytes to be represented in UTF-8. We can get the actual length of the string, based on the number of characters, by using the `String.length/1` function: - -```iex -iex> String.length("hellö") -5 -``` - -The [String module](https://hexdocs.pm/elixir/String.html) contains a bunch of functions that operate on strings as defined in the Unicode standard: - -```iex -iex> String.upcase("hellö") -"HELLÖ" -``` - -## Anonymous functions - -Functions are delimited by the keywords `fn` and `end`: - -```iex -iex> add = fn a, b -> a + b end -#Function<12.71889879/2 in :erl_eval.expr/5> -iex> is_function(add) -true -iex> is_function(add, 2) -true -iex> is_function(add, 1) -false -iex> add.(1, 2) -3 -``` - -Functions are "first class citizens" in Elixir meaning they can be passed as arguments to other functions as integers and strings can. In the example, we have passed the function in the variable `add` to the `is_function/1` function which correctly returned `true`. We can also check the arity of the function by calling `is_function/2`. - -Note a dot (`.`) between the variable and parenthesis is required to invoke an anonymous function. - -Anonymous functions are closures and as such they can access variables that are in scope when the function is defined. Let's define a new anonymous function that uses the `add` anonymous function we have previously defined: - -```iex -iex> double = fn a -> add.(a, a) end -#Function<6.71889879/1 in :erl_eval.expr/5> -iex> double.(2) -4 -``` - -Keep in mind a variable assigned inside a function does not affect its surrounding environment: - -```iex -iex> x = 42 -42 -iex> (fn -> x = 0 end).() -0 -iex> x -42 -``` - -The capture syntax [`&()`](https://hexdocs.pm/elixir/Kernel.SpecialForms.html) can also be used for creating anonymous functions. This type of syntax will be discussed in [Chapter 8](/getting-started/modules.html). - -## (Linked) Lists - -Elixir uses square brackets to specify a list of values. Values can be of any type: - -```iex -iex> [1, 2, true, 3] -[1, 2, true, 3] -iex> length [1, 2, 3] -3 -``` - -Two lists can be concatenated and subtracted using the `++/2` and `--/2` operators: - -```iex -iex> [1, 2, 3] ++ [4, 5, 6] -[1, 2, 3, 4, 5, 6] -iex> [1, true, 2, false, 3, true] -- [true, false] -[1, 2, 3, true] -``` - -Throughout the tutorial, we will talk a lot about the head and tail of a list. The head is the first element of a list and the tail is the remainder of a list. They can be retrieved with the functions `hd/1` and `tl/1`. Let's assign a list to a variable and retrieve its head and tail: - -```iex -iex> list = [1, 2, 3] -iex> hd(list) -1 -iex> tl(list) -[2, 3] -``` - -Getting the head or the tail of an empty list is an error: - -```iex -iex> hd [] -** (ArgumentError) argument error -``` - -Sometimes you will create a list and it will return a value in single-quotes. For example: - -```iex -iex> [11, 12, 13] -'\v\f\r' -iex> [104, 101, 108, 108, 111] -'hello' -``` - -When Elixir sees a list of printable ASCII numbers, Elixir will print that as a char list (literally a list of characters). Char lists are quite common when interfacing with existing Erlang code. Whenever you see a value in IEx and you are not quite sure what it is, you can use the `i/1` to retrieve information about it: - -```iex -iex> i 'hello' -Term - 'hello' -Data type - List -Description - ... -Raw representation - [104, 101, 108, 108, 111] -Reference modules - List -``` - -Keep in mind single-quoted and double-quoted representations are not equivalent in Elixir as they are represented by different types: - -```iex -iex> 'hello' == "hello" -false -``` - -Single-quotes are char lists, double-quotes are strings. We will talk more about them in the ["Binaries, strings and char lists"](/getting-started/binaries-strings-and-char-lists.html) chapter. - -## Tuples - -Elixir uses curly brackets to define tuples. Like lists, tuples can hold any value: - -```iex -iex> {:ok, "hello"} -{:ok, "hello"} -iex> tuple_size {:ok, "hello"} -2 -``` - -Tuples store elements contiguously in memory. This means accessing a tuple element per index or getting the tuple size is a fast operation. Indexes start from zero: - -```iex -iex> tuple = {:ok, "hello"} -{:ok, "hello"} -iex> elem(tuple, 1) -"hello" -iex> tuple_size(tuple) -2 -``` - -It is also possible to put an element at a particular index in a tuple with `put_elem/3`: - -```iex -iex> tuple = {:ok, "hello"} -{:ok, "hello"} -iex> put_elem(tuple, 1, "world") -{:ok, "world"} -iex> tuple -{:ok, "hello"} -``` - -Notice that `put_elem/3` returned a new tuple. The original tuple stored in the `tuple` variable was not modified because Elixir data types are immutable. By being immutable, Elixir code is easier to reason about as you never need to worry if a particular code is mutating your data structure in place. - -## Lists or tuples? - -What is the difference between lists and tuples? - -Lists are stored in memory as linked lists, meaning that each element in a list holds its value and points to the following element until the end of the list is reached. We call each pair of value and pointer a **cons cell**: - -```iex -iex> list = [1 | [2 | [3 | []]]] -[1, 2, 3] -``` - -This means accessing the length of a list is a linear operation: we need to traverse the whole list in order to figure out its size. Updating a list is fast as long as we are prepending elements: - -```iex -iex> [0 | list] -[0, 1, 2, 3] -``` - -Tuples, on the other hand, are stored contiguously in memory. This means getting the tuple size or accessing an element by index is fast. However, updating or adding elements to tuples is expensive because it requires copying the whole tuple in memory. - -Those performance characteristics dictate the usage of those data structures. One very common use case for tuples is to use them to return extra information from a function. For example, `File.read/1` is a function that can be used to read file contents and it returns tuples: - -```iex -iex> File.read("path/to/existing/file") -{:ok, "... contents ..."} -iex> File.read("path/to/unknown/file") -{:error, :enoent} -``` - -If the path given to `File.read/1` exists, it returns a tuple with the atom `:ok` as the first element and the file contents as the second. Otherwise, it returns a tuple with `:error` and the error description. - -Most of the time, Elixir is going to guide you to do the right thing. For example, there is an `elem/2` function to access a tuple item but there is no built-in equivalent for lists: - -```iex -iex> tuple = {:ok, "hello"} -{:ok, "hello"} -iex> elem(tuple, 1) -"hello" -``` - -When "counting" the number of elements in a data structure, Elixir also abides by a simple rule: the function is named `size` if the operation is in constant time (i.e. the value is pre-calculated) or `length` if the operation is linear (i.e. calculating the length gets slower as the input grows). As a mnemonic, both "length" and "linear" start with "l". - -For example, we have used 4 counting functions so far: `byte_size/1` (for the number of bytes in a string), `tuple_size/1` (for the tuple size), `length/1` (for the list length) and `String.length/1` (for the number of graphemes in a string). That said, we use `byte_size` to get the number of bytes in a string, which is cheap, but retrieving the number of unicode characters uses `String.length`, which may be expensive since the whole string needs to be traversed. - -Elixir also provides `Port`, `Reference`, and `PID` as data types (usually used in process communication), and we will take a quick look at them when talking about processes. For now, let's take a look at some of the basic operators that go with our basic types. diff --git a/getting-started/binaries-strings-and-char-lists.markdown b/getting-started/binaries-strings-and-char-lists.markdown index 83b450c23..1836f0b2e 100644 --- a/getting-started/binaries-strings-and-char-lists.markdown +++ b/getting-started/binaries-strings-and-char-lists.markdown @@ -1,198 +1,5 @@ --- -layout: getting-started -title: Binaries, strings, and char lists +layout: redirect +sitemap: false +redirect_to: binaries-strings-and-charlists --- - -# {{ page.title }} - -{% include toc.html %} - -In "Basic types", we learned about strings and used the `is_binary/1` function for checks: - -```iex -iex> string = "hello" -"hello" -iex> is_binary(string) -true -``` - -In this chapter, we will understand what binaries are, how they associate with strings, and what a single-quoted value, `'like this'`, means in Elixir. - -## UTF-8 and Unicode - -A string is a UTF-8 encoded binary. In order to understand exactly what we mean by that, we need to understand the difference between bytes and code points. - -The Unicode standard assigns code points to many of the characters we know. For example, the letter `a` has code point `97` while the letter `ł` has code point `322`. When writing the string `"hełło"` to disk, we need to convert this code point to bytes. If we adopted a rule that said one byte represents one code point, we wouldn't be able to write `"hełło"`, because it uses the code point `322` for `ł`, and one byte can only represent a number from `0` to `255`. But of course, given you can actually read `"hełło"` on your screen, it must be represented *somehow*. That's where encodings come in. - -When representing code points in bytes, we need to encode them somehow. Elixir chose the UTF-8 encoding as its main and default encoding. When we say a string is a UTF-8 encoded binary, we mean a string is a bunch of bytes organized in a way to represent certain code points, as specified by the UTF-8 encoding. - -Since we have characters like `ł` assigned to the code point `322`, we actually need more than one byte to represent them. That's why we see a difference when we calculate the `byte_size/1` of a string compared to its `String.length/1`: - -```iex -iex> string = "hełło" -"hełło" -iex> byte_size(string) -7 -iex> String.length(string) -5 -``` - -There, `byte_size/1` counts the underlying raw bytes, and `String.length/1` counts characters. - -> Note: if you are running on Windows, there is a chance your terminal does not use UTF-8 by default. You can change the encoding of your current session by running `chcp 65001` before entering `iex` (`iex.bat`). - -UTF-8 requires one byte to represent the characters `h`, `e`, and `o`, but two bytes to represent `ł`. In Elixir, you can get a character's code point by using `?`: - -```iex -iex> ?a -97 -iex> ?ł -322 -``` - -You can also use the functions in [the `String` module](https://hexdocs.pm/elixir/String.html) to split a string in its individual characters, each one as a string of length 1: - -```iex -iex> String.codepoints("hełło") -["h", "e", "ł", "ł", "o"] -``` - -You will see that Elixir has excellent support for working with strings. It also supports many of the Unicode operations. In fact, Elixir passes all the tests showcased in the article ["The string type is broken"](http://mortoray.com/2013/11/27/the-string-type-is-broken/). - -However, strings are just part of the story. If a string is a binary, and we have used the `is_binary/1` function, Elixir must have an underlying type empowering strings. And it does! Let's talk about binaries. - -## Binaries (and bitstrings) - -In Elixir, you can define a binary using `<<>>`: - -```iex -iex> <<0, 1, 2, 3>> -<<0, 1, 2, 3>> -iex> byte_size(<<0, 1, 2, 3>>) -4 -``` - -A binary is a sequence of bytes. Those bytes can be organized in any way, even in a sequence that does not make them a valid string: - -```iex -iex> String.valid?(<<239, 191, 191>>) -false -``` - -The string concatenation operation is actually a binary concatenation operator: - -```iex -iex> <<0, 1>> <> <<2, 3>> -<<0, 1, 2, 3>> -``` - -A common trick in Elixir is to concatenate the null byte `<<0>>` to a string to see its inner binary representation: - -```iex -iex> "hełło" <> <<0>> -<<104, 101, 197, 130, 197, 130, 111, 0>> -``` - -Each number given to a binary is meant to represent a byte and therefore must go up to 255. Binaries allow modifiers to be given to store numbers bigger than 255 or to convert a code point to its UTF-8 representation: - -```iex -iex> <<255>> -<<255>> -iex> <<256>> # truncated -<<0>> -iex> <<256 :: size(16)>> # use 16 bits (2 bytes) to store the number -<<1, 0>> -iex> <<256 :: utf8>> # the number is a code point -"Ā" -iex> <<256 :: utf8, 0>> -<<196, 128, 0>> -``` - -If a byte has 8 bits, what happens if we pass a size of 1 bit? - -```iex -iex> <<1 :: size(1)>> -<<1::size(1)>> -iex> <<2 :: size(1)>> # truncated -<<0::size(1)>> -iex> is_binary(<<1 :: size(1)>>) -false -iex> is_bitstring(<<1 :: size(1)>>) -true -iex> bit_size(<< 1 :: size(1)>>) -1 -``` - -The value is no longer a binary, but a bitstring -- a bunch of bits! So a binary is a bitstring where the number of bits is divisible by 8. - -```iex -iex> is_binary(<<1 :: size(16)>>) -true -iex> is_binary(<<1 :: size(15)>>) -false -``` - -We can also pattern match on binaries / bitstrings: - -```iex -iex> <<0, 1, x>> = <<0, 1, 2>> -<<0, 1, 2>> -iex> x -2 -iex> <<0, 1, x>> = <<0, 1, 2, 3>> -** (MatchError) no match of right hand side value: <<0, 1, 2, 3>> -``` - -Note each entry in the binary pattern is expected to match exactly 8 bits. If we want to match on a binary of unknown size, it is possible by using the binary modifier at the end of the pattern: - -```iex -iex> <<0, 1, x :: binary>> = <<0, 1, 2, 3>> -<<0, 1, 2, 3>> -iex> x -<<2, 3>> -``` - -Similar results can be achieved with the string concatenation operator `<>`: - -```iex -iex> "he" <> rest = "hello" -"hello" -iex> rest -"llo" -``` - -A complete reference about the binary / bitstring constructor `<<>>` can be found [in the Elixir documentation](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#%3C%3C%3E%3E/1). This concludes our tour of bitstrings, binaries and strings. A string is a UTF-8 encoded binary and a binary is a bitstring where the number of bits is divisible by 8. Although this shows the flexibility Elixir provides for working with bits and bytes, 99% of the time you will be working with binaries and using the `is_binary/1` and `byte_size/1` functions. - -## Char lists - -A char list is nothing more than a list of code points. Char lists may be created with single-quoted literals: - -```iex -iex> 'hełło' -[104, 101, 322, 322, 111] -iex> is_list 'hełło' -true -iex> 'hello' -'hello' -iex> List.first('hello') -104 -``` - -You can see that, instead of containing bytes, a char list contains the code points of the characters between single-quotes (note that by default IEx will only output code points if any of the integers is outside the ASCII range). So while double-quotes represent a string (i.e. a binary), single-quotes represent a char list (i.e. a list). - -In practice, char lists are used mostly when interfacing with Erlang, in particular old libraries that do not accept binaries as arguments. You can convert a char list to a string and back by using the `to_string/1` and `to_charlist/1` functions: - -```iex -iex> to_charlist "hełło" -[104, 101, 322, 322, 111] -iex> to_string 'hełło' -"hełło" -iex> to_string :hello -"hello" -iex> to_string 1 -"1" -``` - -Note that those functions are polymorphic. They not only convert char lists to strings, but also integers to strings, atoms to strings, and so on. - -With binaries, strings, and char lists out of the way, it is time to talk about key-value data structures. diff --git a/getting-started/case-cond-and-if.markdown b/getting-started/case-cond-and-if.markdown index a0585398c..98dd9f544 100644 --- a/getting-started/case-cond-and-if.markdown +++ b/getting-started/case-cond-and-if.markdown @@ -1,284 +1,5 @@ --- -layout: getting-started -title: case, cond, and if +layout: redirect +sitemap: false +redirect_to: case-cond-and-if --- - -# {{ page.title }} - -{% include toc.html %} - -In this chapter, we will learn about the `case`, `cond`, and `if` control flow structures. - -## `case` - -`case` allows us to compare a value against many patterns until we find a matching one: - -```iex -iex> case {1, 2, 3} do -...> {4, 5, 6} -> -...> "This clause won't match" -...> {1, x, 3} -> -...> "This clause will match and bind x to 2 in this clause" -...> _ -> -...> "This clause would match any value" -...> end -"This clause will match and bind x to 2 in this clause" -``` - -If you want to pattern match against an existing variable, you need to use the `^` operator: - -```iex -iex> x = 1 -1 -iex> case 10 do -...> ^x -> "Won't match" -...> _ -> "Will match" -...> end -"Will match" -``` - -Clauses also allow extra conditions to be specified via guards: - -```iex -iex> case {1, 2, 3} do -...> {1, x, 3} when x > 0 -> -...> "Will match" -...> _ -> -...> "Would match, if guard condition were not satisfied" -...> end -"Will match" -``` - -The first clause above will only match when `x` is positive. - -## Expressions in guard clauses - -Elixir imports and allows the following expressions in guards by default: - -* comparison operators (`==`, `!=`, `===`, `!==`, `>`, `>=`, `<`, `<=`) -* boolean operators (`and`, `or`, `not`) -* arithmetic operations (`+`, `-`, `*`, `/`) -* arithmetic unary operators (`+`, `-`) -* the binary concatenation operator `<>` -* the `in` operator as long as the right side is a range or a list -* all the following type check functions: - * `is_atom/1` - * `is_binary/1` - * `is_bitstring/1` - * `is_boolean/1` - * `is_float/1` - * `is_function/1` - * `is_function/2` - * `is_integer/1` - * `is_list/1` - * `is_map/1` - * `is_nil/1` - * `is_number/1` - * `is_pid/1` - * `is_port/1` - * `is_reference/1` - * `is_tuple/1` -* plus these functions: - * `abs(number)` - * `binary_part(binary, start, length)` - * `bit_size(bitstring)` - * `byte_size(bitstring)` - * `div(integer, integer)` - * `elem(tuple, n)` - * `hd(list)` - * `length(list)` - * `map_size(map)` - * `node()` - * `node(pid | ref | port)` - * `rem(integer, integer)` - * `round(number)` - * `self()` - * `tl(list)` - * `trunc(number)` - * `tuple_size(tuple)` - -Additionally, users may define their own guards. For example, the `Bitwise` -module defines guards as functions and operators: `bnot`, `~~~`, `band`, -`&&&`, `bor`, `|||`, `bxor`, `^^^`, `bsl`, `<<<`, `bsr`, `>>>`. - -Note that while boolean operators such as `and`, `or`, `not` are allowed in guards, -the more general operators `&&`, `||`, and `!` are not. - -Keep in mind errors in guards do not leak but instead make the guard fail: - -```iex -iex> hd(1) -** (ArgumentError) argument error -iex> case 1 do -...> x when hd(x) -> "Won't match" -...> x -> "Got #{x}" -...> end -"Got 1" -``` - -If none of the clauses match, an error is raised: - -```iex -iex> case :ok do -...> :error -> "Won't match" -...> end -** (CaseClauseError) no case clause matching: :ok -``` - -Note anonymous functions can also have multiple clauses and guards: - -```iex -iex> f = fn -...> x, y when x > 0 -> x + y -...> x, y -> x * y -...> end -#Function<12.71889879/2 in :erl_eval.expr/5> -iex> f.(1, 3) -4 -iex> f.(-1, 3) --3 -``` - -The number of arguments in each anonymous function clause needs to be the same, otherwise an error is raised. - -```iex -iex> f2 = fn -...> x, y when x > 0 -> x + y -...> x, y, z -> x * y + z -...> end -** (CompileError) iex:1: cannot mix clauses with different arities in function definition -``` - -## `cond` - -`case` is useful when you need to match against different values. However, in many circumstances, we want to check different conditions and find the first one that evaluates to true. In such cases, one may use `cond`: - -```iex -iex> cond do -...> 2 + 2 == 5 -> -...> "This will not be true" -...> 2 * 2 == 3 -> -...> "Nor this" -...> 1 + 1 == 2 -> -...> "But this will" -...> end -"But this will" -``` - -This is equivalent to `else if` clauses in many imperative languages (although used way less frequently here). - -If none of the conditions return true, an error (`CondClauseError`) is raised. For this reason, it may be necessary to add a final condition, equal to `true`, which will always match: - -```iex -iex> cond do -...> 2 + 2 == 5 -> -...> "This is never true" -...> 2 * 2 == 3 -> -...> "Nor this" -...> true -> -...> "This is always true (equivalent to else)" -...> end -"This is always true (equivalent to else)" -``` - -Finally, note `cond` considers any value besides `nil` and `false` to be true: - -```iex -iex> cond do -...> hd([1, 2, 3]) -> -...> "1 is considered as true" -...> end -"1 is considered as true" -``` - -## `if` and `unless` - -Besides `case` and `cond`, Elixir also provides the macros `if/2` and `unless/2` which are useful when you need to check for only one condition: - -```iex -iex> if true do -...> "This works!" -...> end -"This works!" -iex> unless true do -...> "This will never be seen" -...> end -nil -``` - -If the condition given to `if/2` returns `false` or `nil`, the body given between `do/end` is not executed and instead it returns `nil`. The opposite happens with `unless/2`. - -They also support `else` blocks: - -```iex -iex> if nil do -...> "This won't be seen" -...> else -...> "This will" -...> end -"This will" -``` - -> Note: An interesting note regarding `if/2` and `unless/2` is that they are implemented as macros in the language; they aren't special language constructs as they would be in many languages. You can check the documentation and the source of `if/2` in [the `Kernel` module docs](https://hexdocs.pm/elixir/Kernel.html). The `Kernel` module is also where operators like `+/2` and functions like `is_function/2` are defined, all automatically imported and available in your code by default. - -## `do/end` blocks - -At this point, we have learned four control structures: `case`, `cond`, `if`, and `unless`, and they were all wrapped in `do/end` blocks. It happens we could also write `if` as follows: - -```iex -iex> if true, do: 1 + 2 -3 -``` - -Notice how the example above has a comma between `true` and `do:`, that's because it is using Elixir's regular syntax where each argument is separated by comma. We say this syntax is using *keyword lists*. We can pass `else` using keywords too: - -```iex -iex> if false, do: :this, else: :that -:that -``` - -`do/end` blocks are a syntactic convenience built on top of the keywords one. That's why `do/end` blocks do not require a comma between the previous argument and the block. They are useful exactly because they remove the verbosity when writing blocks of code. These are equivalent: - -```iex -iex> if true do -...> a = 1 + 2 -...> a + 10 -...> end -13 -iex> if true, do: ( -...> a = 1 + 2 -...> a + 10 -...> ) -13 -``` - -One thing to keep in mind when using `do/end` blocks is they are always bound to the outermost function call. For example, the following expression: - -```iex -iex> is_number if true do -...> 1 + 2 -...> end -** (CompileError) undefined function: is_number/2 -``` - -Would be parsed as: - -```iex -iex> is_number(if true) do -...> 1 + 2 -...> end -** (CompileError) undefined function: is_number/2 -``` - -which leads to an undefined function error because that invocation passes two arguments, and `is_number/2` does not exist. The `if true` expression is invalid in itself because it needs the block, but since the arity of `is_number/2` does not match, Elixir does not even reach its evaluation. - -Adding explicit parentheses is enough to bind the block to `if`: - -```iex -iex> is_number(if true do -...> 1 + 2 -...> end) -true -``` - -Keyword lists play an important role in the language and are quite common in many functions and macros. We will explore them a bit more in a future chapter. Now it is time to talk about "Binaries, strings, and char lists". diff --git a/getting-started/comprehensions.markdown b/getting-started/comprehensions.markdown index 440b20819..a1ddb4dd7 100644 --- a/getting-started/comprehensions.markdown +++ b/getting-started/comprehensions.markdown @@ -1,155 +1,5 @@ --- -layout: getting-started -title: Comprehensions +layout: redirect +sitemap: false +redirect_to: comprehensions --- - -# {{ page.title }} - -{% include toc.html %} - -In Elixir, it is common to loop over an Enumerable, often filtering out some results and mapping values into another list. Comprehensions are syntactic sugar for such constructs: they group those common tasks into the `for` special form. - -For example, we can map a list of integers into their squared values: - -```iex -iex> for n <- [1, 2, 3, 4], do: n * n -[1, 4, 9, 16] -``` - -A comprehension is made of three parts: generators, filters and collectables. - -## Generators and filters - -In the expression above, `n <- [1, 2, 3, 4]` is the **generator**. It is literally generating values to be used in the comprehension. Any enumerable can be passed in the right-hand side of the generator expression: - -```iex -iex> for n <- 1..4, do: n * n -[1, 4, 9, 16] -``` - -Generator expressions also support pattern matching on their left-hand side; all non-matching patterns are *ignored*. Imagine that, instead of a range, we have a keyword list where the key is the atom `:good` or `:bad` and we only want to compute the square of the `:good` values: - -```iex -iex> values = [good: 1, good: 2, bad: 3, good: 4] -iex> for {:good, n} <- values, do: n * n -[1, 4, 16] -``` - -Alternatively to pattern matching, filters can be used to select some particular elements. For example, we can select the multiples of 3 and discard all others: - -```iex -iex> multiple_of_3? = fn(n) -> rem(n, 3) == 0 end -iex> for n <- 0..5, multiple_of_3?.(n), do: n * n -[0, 9] -``` - -Comprehensions discard all elements for which the filter expression returns `false` or `nil`; all other values are selected. - -Comprehensions generally provide a much more concise representation than using the equivalent functions from the `Enum` and `Stream` modules. Furthermore, comprehensions also allow multiple generators and filters to be given. Here is an example that receives a list of directories and gets the size of each file in those directories: - -```elixir -for dir <- dirs, - file <- File.ls!(dir), - path = Path.join(dir, file), - File.regular?(path) do - File.stat!(path).size -end -``` - -Multiple generators can also be used to calculate the cartesian product of two lists: - -```iex -iex> for i <- [:a, :b, :c], j <- [1, 2], do: {i, j} -[a: 1, a: 2, b: 1, b: 2, c: 1, c: 2] -``` - -A more advanced example of multiple generators and filters is Pythagorean triples. A Pythagorean triple is a set of positive integers such that `a*a + b*b = c*c`, let's write a comprehension in a file named `triple.exs`: - -```elixir -defmodule Triple do - def pythagorean(n) when n > 0 do - for a <- 1..n, - b <- 1..n, - c <- 1..n, - a + b + c <= n, - a*a + b*b == c*c, - do: {a, b, c} - end -end -``` - -Now on terminal: - -```bash -iex triple.exs -``` - -```iex -iex> Triple.pythagorean(5) -[] -iex> Triple.pythagorean(12) -[{3, 4, 5}, {4, 3, 5}] -iex> Triple.pythagorean(48) -[{3, 4, 5}, {4, 3, 5}, {5, 12, 13}, {6, 8, 10}, {8, 6, 10}, {8, 15, 17}, - {9, 12, 15}, {12, 5, 13}, {12, 9, 15}, {12, 16, 20}, {15, 8, 17}, {16, 12, 20}] -``` - -The code above is quite expensive when the range of search is a large number. Additionally, since the tuple `{b, a, c}` represents the same Pythagorean triple as `{a, b, c}`, our function yields duplicate triples. We can optimize the comprehension and eliminate the duplicate results by referencing the variables from previous generators in the following ones, for example: - -```elixir -defmodule Triple do - def pythagorean(n) when n > 0 do - for a <- 1..n-2, - b <- a+1..n-1, - c <- b+1..n, - a + b + c <= n, - a*a + b*b == c*c, - do: {a, b, c} - end -end -``` - -Finally, keep in mind that variable assignments inside the comprehension, be it in generators, filters or inside the block, are not reflected outside of the comprehension. - -## Bitstring generators - -Bitstring generators are also supported and are very useful when you need to comprehend over bitstring streams. The example below receives a list of pixels from a binary with their respective red, green and blue values and converts them into tuples of three elements each: - -```iex -iex> pixels = <<213, 45, 132, 64, 76, 32, 76, 0, 0, 234, 32, 15>> -iex> for <>, do: {r, g, b} -[{213, 45, 132}, {64, 76, 32}, {76, 0, 0}, {234, 32, 15}] -``` - -A bitstring generator can be mixed with "regular" enumerable generators, and supports filters as well. - -## The `:into` option - -In the examples above, all the comprehensions returned lists as their result. However, the result of a comprehension can be inserted into different data structures by passing the `:into` option to the comprehension. - -For example, a bitstring generator can be used with the `:into` option in order to easily remove all spaces in a string: - -```iex -iex> for <>, c != ?\s, into: "", do: <> -"helloworld" -``` - -Sets, maps and other dictionaries can also be given to the `:into` option. In general, `:into` accepts any structure that implements the `Collectable` protocol. - -A common use case of `:into` can be transforming values in a map, without touching the keys: - -```iex -iex> for {key, val} <- %{"a" => 1, "b" => 2}, into: %{}, do: {key, val * val} -%{"a" => 1, "b" => 4} -``` - -Let's make another example using streams. Since the `IO` module provides streams (that are both `Enumerable`s and `Collectable`s), an echo terminal that echoes back the upcased version of whatever is typed can be implemented using comprehensions: - -```iex -iex> stream = IO.stream(:stdio, :line) -iex> for line <- stream, into: stream do -...> String.upcase(line) <> "\n" -...> end -``` - -Now type any string into the terminal and you will see that the same value will be printed in upper-case. Unfortunately, this example also got your IEx shell stuck in the comprehension, so you will need to hit `Ctrl+C` twice to get out of it. :) diff --git a/getting-started/debugging.markdown b/getting-started/debugging.markdown new file mode 100644 index 000000000..c63f3e447 --- /dev/null +++ b/getting-started/debugging.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: debugging +--- diff --git a/getting-started/enumerables-and-streams.markdown b/getting-started/enumerables-and-streams.markdown index 608e7fe32..b6abedae7 100644 --- a/getting-started/enumerables-and-streams.markdown +++ b/getting-started/enumerables-and-streams.markdown @@ -1,124 +1,5 @@ --- -layout: getting-started -title: Enumerables and Streams +layout: redirect +sitemap: false +redirect_to: enumerable-and-streams --- - -# {{ page.title }} - -{% include toc.html %} - -## Enumerables - -Elixir provides the concept of enumerables and [the `Enum` module](https://hexdocs.pm/elixir/Enum.html) to work with them. We have already learned two enumerables: lists and maps. - -```iex -iex> Enum.map([1, 2, 3], fn x -> x * 2 end) -[2, 4, 6] -iex> Enum.map(%{1 => 2, 3 => 4}, fn {k, v} -> k * v end) -[2, 12] -``` - -The `Enum` module provides a huge range of functions to transform, sort, group, filter and retrieve items from enumerables. It is one of the modules developers use frequently in their Elixir code. - -Elixir also provides ranges: - -```iex -iex> Enum.map(1..3, fn x -> x * 2 end) -[2, 4, 6] -iex> Enum.reduce(1..3, 0, &+/2) -6 -``` - -The functions in the Enum module are limited to, as the name says, enumerating values in data structures. For specific operations, like inserting and updating particular elements, you may need to reach for modules specific to the data type. For example, if you want to insert an element at a given position in a list, you should use the `List.insert_at/3` function from [the `List` module](https://hexdocs.pm/elixir/List.html), as it would make little sense to insert a value into, for example, a range. - -We say the functions in the `Enum` module are polymorphic because they can work with diverse data types. In particular, the functions in the `Enum` module can work with any data type that implements [the `Enumerable` protocol](https://hexdocs.pm/elixir/Enumerable.html). We are going to discuss Protocols in a later chapter; for now we are going to move on to a specific kind of enumerable called a stream. - -## Eager vs Lazy - -All the functions in the `Enum` module are eager. Many functions expect an enumerable and return a list back: - -```iex -iex> odd? = &(rem(&1, 2) != 0) -#Function<6.80484245/1 in :erl_eval.expr/5> -iex> Enum.filter(1..3, odd?) -[1, 3] -``` - -This means that when performing multiple operations with `Enum`, each operation is going to generate an intermediate list until we reach the result: - -```iex -iex> 1..100_000 |> Enum.map(&(&1 * 3)) |> Enum.filter(odd?) |> Enum.sum -7500000000 -``` - -The example above has a pipeline of operations. We start with a range and then multiply each element in the range by 3. This first operation will now create and return a list with `100_000` items. Then we keep all odd elements from the list, generating a new list, now with `50_000` items, and then we sum all entries. - -## The pipe operator - -The `|>` symbol used in the snippet above is the **pipe operator**: it takes the output from the expression on its left side and passes it as the first argument to the function call on its right side. It's similar to the Unix `|` operator. Its purpose is to highlight the data being transformed by a series of functions. To see how it can make the code cleaner, have a look at the example above rewritten without using the `|>` operator: - -```iex -iex> Enum.sum(Enum.filter(Enum.map(1..100_000, &(&1 * 3)), odd?)) -7500000000 -``` - -Find more about the pipe operator [by reading its documentation](https://hexdocs.pm/elixir/Kernel.html#%7C%3E/2). - -## Streams - -As an alternative to `Enum`, Elixir provides [the `Stream` module](https://hexdocs.pm/elixir/Stream.html) which supports lazy operations: - -```iex -iex> 1..100_000 |> Stream.map(&(&1 * 3)) |> Stream.filter(odd?) |> Enum.sum -7500000000 -``` - -Streams are lazy, composable enumerables. - -In the example above, `1..100_000 |> Stream.map(&(&1 * 3))` returns a data type, an actual stream, that represents the `map` computation over the range `1..100_000`: - -```iex -iex> 1..100_000 |> Stream.map(&(&1 * 3)) -#Stream<[enum: 1..100000, funs: [#Function<34.16982430/1 in Stream.map/2>]]> -``` - -Furthermore, they are composable because we can pipe many stream operations: - -```iex -iex> 1..100_000 |> Stream.map(&(&1 * 3)) |> Stream.filter(odd?) -#Stream<[enum: 1..100000, funs: [...]]> -``` - -Instead of generating intermediate lists, streams build a series of computations that are invoked only when we pass the underlying stream to the `Enum` module. Streams are useful when working with large, *possibly infinite*, collections. - -Many functions in the `Stream` module accept any enumerable as an argument and return a stream as a result. It also provides functions for creating streams. For example, `Stream.cycle/1` can be used to create a stream that cycles a given enumerable infinitely. Be careful to not call a function like `Enum.map/2` on such streams, as they would cycle forever: - -```iex -iex> stream = Stream.cycle([1, 2, 3]) -#Function<15.16982430/2 in Stream.cycle/1> -iex> Enum.take(stream, 10) -[1, 2, 3, 1, 2, 3, 1, 2, 3, 1] -``` - -On the other hand, `Stream.unfold/2` can be used to generate values from a given initial value: - -```iex -iex> stream = Stream.unfold("hełło", &String.next_codepoint/1) -#Function<39.75994740/2 in Stream.unfold/2> -iex> Enum.take(stream, 3) -["h", "e", "ł"] -``` - -Another interesting function is `Stream.resource/3` which can be used to wrap around resources, guaranteeing they are opened right before enumeration and closed afterwards, even in the case of failures. For example, we can use it to stream a file: - -```iex -iex> stream = File.stream!("path/to/file") -#Function<18.16982430/2 in Stream.resource/3> -iex> Enum.take(stream, 10) -``` - -The example above will fetch the first 10 lines of the file you have selected. This means streams can be very useful for handling large files or even slow resources like network resources. - -The amount of functionality in the [`Enum`](https://hexdocs.pm/elixir/Enum.html) and [`Stream`](https://hexdocs.pm/elixir/Stream.html) modules can be daunting at first, but you will get familiar with them case by case. In particular, focus on the `Enum` module first and only move to `Stream` for the particular scenarios where laziness is required, to either deal with slow resources or large, possibly infinite, collections. - -Next we'll look at a feature central to Elixir, Processes, which allows us to write concurrent, parallel and distributed programs in an easy and understandable way. diff --git a/getting-started/erlang-libraries.markdown b/getting-started/erlang-libraries.markdown index 6226c3409..8af923cc7 100644 --- a/getting-started/erlang-libraries.markdown +++ b/getting-started/erlang-libraries.markdown @@ -1,203 +1,5 @@ --- -layout: getting-started -title: Erlang libraries +layout: redirect +sitemap: false +redirect_to: erlang-libraries --- - -# {{ page.title }} - -{% include toc.html %} - -Elixir provides excellent interoperability with Erlang libraries. In fact, -Elixir discourages simply wrapping Erlang libraries in favor of directly -interfacing with Erlang code. In this section we will present some of the -most common and useful Erlang functionality that is not found in Elixir. - -As you grow more proficient in Elixir, you may want to explore the Erlang -[STDLIB Reference Manual](http://erlang.org/doc/apps/stdlib/index.html) in more -detail. - -## The binary module - -The built-in Elixir String module handles binaries that are UTF-8 encoded. -[The binary module](http://erlang.org/doc/man/binary.html) is useful when -you are dealing with binary data that is not necessarily UTF-8 encoded. - -```iex -iex> String.to_charlist "Ø" -[216] -iex> :binary.bin_to_list "Ø" -[195, 152] -``` - -The above example shows the difference; the `String` module returns UTF-8 -codepoints, while `:binary` deals with raw data bytes. - -## Formatted text output - -Elixir does not contain a function similar to `printf` found in C and other -languages. Luckily, the Erlang standard library functions `:io.format/2` and -`:io_lib.format/2` may be used. The first formats to terminal output, while -the second formats to an iolist. The format specifiers differ from `printf`, -[refer to the Erlang documentation for details](http://erlang.org/doc/man/io.html#format-1). - -```iex -iex> :io.format("Pi is approximately given by:~10.3f~n", [:math.pi]) -Pi is approximately given by: 3.142 -:ok -iex> to_string :io_lib.format("Pi is approximately given by:~10.3f~n", [:math.pi]) -"Pi is approximately given by: 3.142\n" -``` - -Also note that Erlang's formatting functions require special attention to -Unicode handling. - -## The crypto module - -[The crypto module](http://erlang.org/doc/man/crypto.html) contains hashing -functions, digital signatures, encryption and more: - -```iex -iex> Base.encode16(:crypto.hash(:sha256, "Elixir")) -"3315715A7A3AD57428298676C5AE465DADA38D951BDFAC9348A8A31E9C7401CB" -``` - -The `:crypto` module is not part of the Erlang standard library, but is -included with the Erlang distribution. This means you must list `:crypto` -in your project's applications list whenever you use it. To do this, -edit your `mix.exs` file to include: - -```elixir -def application do - [applications: [:crypto]] -end -``` - -## The digraph module - -[The digraph module](http://erlang.org/doc/man/digraph.html) (as well as -[digraph_utils](http://erlang.org/doc/man/digraph_utils.html)) contains -functions for dealing with directed graphs built of vertices and edges. -After constructing the graph, the algorithms in there will help finding -for instance the shortest path between two vertices, or loops in the graph. - -Note that the functions in `:digraph` alter the graph structure indirectly -as a side effect, while returning the added vertices or edges. - -Given three vertices, find the shortest path from the first to the last. - -```iex -iex> digraph = :digraph.new() -iex> coords = [{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}] -iex> [v0, v1, v2] = (for c <- coords, do: :digraph.add_vertex(digraph, c)) -iex> :digraph.add_edge(digraph, v0, v1) -iex> :digraph.add_edge(digraph, v1, v2) -iex> :digraph.get_short_path(digraph, v0, v2) -[{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}] -``` - -## Erlang Term Storage - -The modules [`ets`](http://erlang.org/doc/man/ets.html) and -[`dets`](http://erlang.org/doc/man/dets.html) handle storage of large -data structures in memory or on disk respectively. - -ETS lets you create a table containing tuples. By default, ETS tables -are protected, which means only the owner process may write to the table -but any other process can read. ETS has some functionality to be used as -a simple database, a key-value store or as a cache mechanism. - -The functions in the `ets` module will modify the state of the table as a -side-effect. - -```iex -iex> table = :ets.new(:ets_test, []) -# Store as tuples with {name, population} -iex> :ets.insert(table, {"China", 1_374_000_000}) -iex> :ets.insert(table, {"India", 1_284_000_000}) -iex> :ets.insert(table, {"USA", 322_000_000}) -iex> :ets.i(table) -<1 > {"USA", 322000000} -<2 > {"China", 1_374_000_000} -<3 > {"India", 1_284_000_000} -``` - -## The math module - -[The `math` module](http://erlang.org/doc/man/math.html) contains common -mathematical operations covering trigonometry, exponential and logarithmic -functions. - -```iex -iex> angle_45_deg = :math.pi() * 45.0 / 180.0 -iex> :math.sin(angle_45_deg) -0.7071067811865475 -iex> :math.exp(55.0) -7.694785265142018e23 -iex> :math.log(7.694785265142018e23) -55.0 -``` - -## The queue module - -The [`queue` is a data structure](http://erlang.org/doc/man/queue.html) -that implements (double-ended) FIFO (first-in first-out) queues efficiently: - -```iex -iex> q = :queue.new -iex> q = :queue.in("A", q) -iex> q = :queue.in("B", q) -iex> {value, q} = :queue.out(q) -iex> value -{:value, "A"} -iex> {value, q} = :queue.out(q) -iex> value -{:value, "B"} -iex> {value, q} = :queue.out(q) -iex> value -:empty -``` - -## The rand module - -[`rand` has functions](http://erlang.org/doc/man/rand.html) for returning -random values and setting the random seed. - -```iex -iex> :rand.uniform() -0.8175669086010815 -iex> _ = :rand.seed(:exs1024, {123, 123534, 345345}) -iex> :rand.uniform() -0.5820506340260994 -iex> :rand.uniform(6) -6 -``` - -## The zip and zlib modules - -[The `zip` module](http://erlang.org/doc/man/zip.html) lets you read and write zip files to and from disk or memory, -as well as extracting file information. - -This code counts the number of files in a zip file: - -```iex -iex> :zip.foldl(fn _, _, _, acc -> acc + 1 end, 0, :binary.bin_to_list("file.zip")) -{:ok, 633} -``` - -[The `zlib` module](http://erlang.org/doc/man/zlib.html) deals with data compression in zlib format, as found in the -`gzip` command. - -```iex -iex> song = " -...> Mary had a little lamb, -...> His fleece was white as snow, -...> And everywhere that Mary went, -...> The lamb was sure to go." -iex> compressed = :zlib.compress(song) -iex> byte_size song -110 -iex> byte_size compressed -99 -iex> :zlib.uncompress(compressed) -"\nMary had a little lamb,\nHis fleece was white as snow,\nAnd everywhere that Mary went,\nThe lamb was sure to go." -``` diff --git a/getting-started/index.html b/getting-started/index.html index 00b1bd4e3..c50365d75 100644 --- a/getting-started/index.html +++ b/getting-started/index.html @@ -1,5 +1,5 @@ - - - - - +--- +layout: redirect +sitemap: false +redirect_to: introduction +--- diff --git a/getting-started/introduction.markdown b/getting-started/introduction.markdown index c5118a759..c50365d75 100644 --- a/getting-started/introduction.markdown +++ b/getting-started/introduction.markdown @@ -1,82 +1,5 @@ --- -layout: getting-started -title: Introduction -redirect_from: /getting_started/1.html +layout: redirect +sitemap: false +redirect_to: introduction --- - -# {{ page.title }} - -{% include toc.html %} - -Welcome! - -In this tutorial we are going to teach you the Elixir foundation, the language syntax, how to define modules, how to manipulate the characteristics of common data structures, and more. This chapter will focus on ensuring Elixir is installed and that you can successfully run Elixir's Interactive Shell, called IEx. - -Our requirements are: - - * Elixir - Version 1.2.0 onwards - * Erlang - Version 18.0 onwards - -Let's get started! - -> If you find any errors in the tutorial or on the website, [please report a bug or send a pull request to our issue tracker](https://github.com/elixir-lang/elixir-lang.github.com). - -## Installation - -If you still haven't installed Elixir, run to our [installation page](/install.html). Once you are done, you can run `elixir -v` to get the current Elixir version. - -## Interactive mode - -When you install Elixir, you will have three new executables: `iex`, `elixir` and `elixirc`. If you compiled Elixir from source or are using a packaged version, you can find these inside the `bin` directory. - -For now, let's start by running `iex` (or `iex.bat` if you are on Windows) which stands for Interactive Elixir. In interactive mode, we can type any Elixir expression and get its result. Let's warm up with some basic expressions. - -Open up `iex` and type the following expressions: - -```iex -Erlang/OTP 19 [erts-8.1] [source] [64-bit] [smp:4:4] [async-threads:10] [hipe] [kernel-poll:false] [dtrace] - -Interactive Elixir (1.3.4) - press Ctrl+C to exit (type h() ENTER for help) -iex(1)> 40 + 2 -42 -iex(2)> "hello" <> " world" -"hello world" -``` - -Please note that some details like version numbers may differ a bit in your session, that's not important. From now on `iex` sessions will be stripped down to focus on the code. To exit `iex` press `Ctrl+C` twice. - -It seems we are ready to go! We will use the interactive shell quite a lot in the next chapters to get a bit more familiar with the language constructs and basic types, starting in the next chapter. - -> Note: if you are on Windows, you can also try `iex.bat --werl` which may provide a better experience depending on which console you are using. - -## Running scripts - -After getting familiar with the basics of the language you may want to try writing simple programs. This can be accomplished by putting the following Elixir code into a file: - -```elixir -IO.puts "Hello world from Elixir" -``` - -Save it as `simple.exs` and execute it with `elixir`: - -```bash -$ elixir simple.exs -Hello world from Elixir -``` - -Later on we will learn how to compile Elixir code (in [Chapter 8](/getting-started/modules.html)) and how to use the Mix build tool (in the [Mix & OTP guide](/getting-started/mix-otp/introduction-to-mix.html)). For now, let's move on to [Chapter 2](/getting-started/basic-types.html). - -## Asking questions - -When going through this getting started guide, it is common to have questions, after all, that is part of the learning process! There are many places you could ask them to learn more about Elixir: - - * [#elixir-lang on freenode IRC](irc://irc.freenode.net/elixir-lang) - * [Elixir on Slack](https://elixir-slackin.herokuapp.com/) - * [Elixir Forum](http://elixirforum.com) - * [elixir tag on StackOverflow](https://stackoverflow.com/questions/tagged/elixir) - -When asking questions, remember these two tips: - - * Instead of asking "how to do X in Elixir", ask "how to solve Y in Elixir". In other words, don't ask how to implement a particular solution, instead describe the problem at hand. Stating the problem gives more context and less bias for a correct answer. - - * In case things are not working as expected, please include as much information as you can in your report, for example: your Elixir version, the code snippet and the error message alongside the error stacktrace. Use sites like [Gist](https://gist.github.com/) to paste this information. diff --git a/getting-started/io-and-the-file-system.markdown b/getting-started/io-and-the-file-system.markdown index 8abab42aa..17a6e7bef 100644 --- a/getting-started/io-and-the-file-system.markdown +++ b/getting-started/io-and-the-file-system.markdown @@ -1,169 +1,5 @@ --- -layout: getting-started -title: IO and the file system +layout: redirect +sitemap: false +redirect_to: io-and-the-file-system --- - -# {{ page.title }} - -{% include toc.html %} - -This chapter is a quick introduction to input/output mechanisms and file-system-related tasks, as well as to related modules like [`IO`](https://hexdocs.pm/elixir/IO.html), [`File`](https://hexdocs.pm/elixir/File.html) and [`Path`](https://hexdocs.pm/elixir/Path.html). - -We had originally sketched this chapter to come much earlier in the getting started guide. However, we noticed the IO system provides a great opportunity to shed some light on some philosophies and curiosities of Elixir and the VM. - -## The `IO` module - -The `IO` module is the main mechanism in Elixir for reading and writing to standard input/output (`:stdio`), standard error (`:stderr`), files and other IO devices. Usage of the module is pretty straightforward: - -```iex -iex> IO.puts "hello world" -hello world -:ok -iex> IO.gets "yes or no? " -yes or no? yes -"yes\n" -``` - -By default, functions in the IO module read from the standard input and write to the standard output. We can change that by passing, for example, `:stderr` as an argument (in order to write to the standard error device): - -```iex -iex> IO.puts :stderr, "hello world" -hello world -:ok -``` - -## The `File` module - -The [`File`](https://hexdocs.pm/elixir/File.html) module contains functions that allow us to open files as IO devices. By default, files are opened in binary mode, which requires developers to use the specific `IO.binread/2` and `IO.binwrite/2` functions from the `IO` module: - -```iex -iex> {:ok, file} = File.open "hello", [:write] -{:ok, #PID<0.47.0>} -iex> IO.binwrite file, "world" -:ok -iex> File.close file -:ok -iex> File.read "hello" -{:ok, "world"} -``` - -A file can also be opened with `:utf8` encoding, which tells the `File` module to interpret the bytes read from the file as UTF-8-encoded bytes. - -Besides functions for opening, reading and writing files, the `File` module has many functions to work with the file system. Those functions are named after their UNIX equivalents. For example, `File.rm/1` can be used to remove files, `File.mkdir/1` to create directories, `File.mkdir_p/1` to create directories and all their parent chain. There are even `File.cp_r/2` and `File.rm_rf/1` to respectively copy and remove files and directories recursively (i.e., copying and removing the contents of the directories too). - -You will also notice that functions in the `File` module have two variants: one "regular" variant and another variant with a trailing bang (`!`). For example, when we read the `"hello"` file in the example above, we use `File.read/1`. Alternatively, we can use `File.read!/1`: - -```iex -iex> File.read "hello" -{:ok, "world"} -iex> File.read! "hello" -"world" -iex> File.read "unknown" -{:error, :enoent} -iex> File.read! "unknown" -** (File.Error) could not read file unknown: no such file or directory -``` - -Notice that when the file does not exist, the version with `!` raises an error. The version without `!` is preferred when you want to handle different outcomes using pattern matching: - -```elixir -case File.read(file) do - {:ok, body} -> # do something with the `body` - {:error, reason} -> # handle the error caused by `reason` -end -``` - -However, if you expect the file to be there, the bang variation is more useful as it raises a meaningful error message. Avoid writing: - -```elixir -{:ok, body} = File.read(file) -``` - -as, in case of an error, `File.read/1` will return `{:error, reason}` and the pattern matching will fail. You will still get the desired result (a raised error), but the message will be about the pattern which doesn't match (thus being cryptic in respect to what the error actually is about). - -Therefore, if you don't want to handle the error outcomes, prefer using `File.read!/1`. - -## The `Path` module - -The majority of the functions in the `File` module expect paths as arguments. Most commonly, those paths will be regular binaries. The [`Path`](https://hexdocs.pm/elixir/Path.html) module provides facilities for working with such paths: - -```iex -iex> Path.join("foo", "bar") -"foo/bar" -iex> Path.expand("~/hello") -"/Users/jose/hello" -``` - -Using functions from the `Path` module as opposed to directly manipulating strings is preferred since the `Path` module takes care of different operating systems transparently. Finally, keep in mind that Elixir will automatically convert slashes (`/`) into backslashes (`\`) on Windows when performing file operations. - -With this we have covered the main modules that Elixir provides for dealing with IO and interacting with the file system. In the next sections, we will discuss some advanced topics regarding IO. Those sections are not necessary in order to write Elixir code, so feel free to skip them, but they do provide a nice overview of how the IO system is implemented in the VM and other curiosities. - -## Processes and group leaders - -You may have noticed that `File.open/2` returns a tuple like `{:ok, pid}`: - -```iex -iex> {:ok, file} = File.open "hello", [:write] -{:ok, #PID<0.47.0>} -``` - -That happens because the `IO` module actually works with processes (see [chapter 11](/getting-started/processes.html)). When you write `IO.write(pid, binary)`, the `IO` module will send a message to the process identified by `pid` with the desired operation. Let's see what happens if we use our own process: - -```iex -iex> pid = spawn fn -> -...> receive do: (msg -> IO.inspect msg) -...> end -#PID<0.57.0> -iex> IO.write(pid, "hello") -{:io_request, #PID<0.41.0>, #Reference<0.0.8.91>, {:put_chars, :unicode, "hello"}} -** (ErlangError) erlang error: :terminated -``` - -After `IO.write/2`, we can see the request sent by the `IO` module (a four-elements tuple) printed out. Soon after that, we see that it fails since the `IO` module expected some kind of result that we did not supply. - -The [`StringIO`](https://hexdocs.pm/elixir/StringIO.html) module provides an implementation of the `IO` device messages on top of strings: - -```iex -iex> {:ok, pid} = StringIO.open("hello") -{:ok, #PID<0.43.0>} -iex> IO.read(pid, 2) -"he" -``` - -By modelling IO devices with processes, the Erlang VM allows different nodes in the same network to exchange file processes in order to read/write files in between nodes. Of all IO devices, there is one that is special to each process: the **group leader**. - -When you write to `:stdio`, you are actually sending a message to the group leader, which writes to the standard-output file descriptor: - -```iex -iex> IO.puts :stdio, "hello" -hello -:ok -iex> IO.puts Process.group_leader, "hello" -hello -:ok -``` - -The group leader can be configured per process and is used in different situations. For example, when executing code in a remote terminal, it guarantees messages in a remote node are redirected and printed in the terminal that triggered the request. - -## `iodata` and `chardata` - -In all of the examples above, we used binaries when writing to files. In the chapter ["Binaries, strings and char lists"](/getting-started/binaries-strings-and-char-lists.html), we mentioned how strings are made of bytes while char lists are lists with unicode codepoints. - -The functions in `IO` and `File` also allow lists to be given as arguments. Not only that, they also allow a mixed list of lists, integers and binaries to be given: - -```iex -iex> IO.puts 'hello world' -hello world -:ok -iex> IO.puts ['hello', ?\s, "world"] -hello world -:ok -``` - -However, using list in IO operations requires some attention. A list may represent either a bunch of bytes or a bunch of characters and which one to use depends on the encoding of the IO device. If the file is opened without encoding, the file is expected to be in raw mode, and the functions in the `IO` module starting with `bin*` must be used. Those functions expect an `iodata` as argument; i.e., they expect a list of integers representing bytes and binaries to be given. - -On the other hand, `:stdio` and files opened with `:utf8` encoding work with the remaining functions in the `IO` module. Those functions expect a `char_data` as an argument, that is, a list of characters or strings. - -Although this is a subtle difference, you only need to worry about these details if you intend to pass lists to those functions. Binaries are already represented by the underlying bytes and as such their representation is always "raw". - -This finishes our tour of IO devices and IO related functionality. We have learned about four Elixir modules - [`IO`](https://hexdocs.pm/elixir/IO.html), [`File`](https://hexdocs.pm/elixir/File.html), [`Path`](https://hexdocs.pm/elixir/Path.html) and [`StringIO`](https://hexdocs.pm/elixir/StringIO.html) - as well as how the VM uses processes for the underlying IO mechanisms and how to use `chardata` and `iodata` for IO operations. diff --git a/getting-started/keywords-and-maps.markdown b/getting-started/keywords-and-maps.markdown index df389346e..a7430b486 100644 --- a/getting-started/keywords-and-maps.markdown +++ b/getting-started/keywords-and-maps.markdown @@ -1,224 +1,5 @@ --- -layout: getting-started -title: Keywords and maps -redirect_from: /getting-started/maps-and-dicts.html +layout: redirect +sitemap: false +redirect_to: keywords-and-maps --- - -# {{ page.title }} - -{% include toc.html %} - -So far we haven't discussed any associative data structures, i.e. data structures that are able to associate a certain value (or multiple values) to a key. Different languages call these different names like dictionaries, hashes, associative arrays, etc. - -In Elixir, we have two main associative data structures: keyword lists and maps. It's time to learn more about them! - -## Keyword lists - -In many functional programming languages, it is common to use a list of 2-item tuples as the representation of a key-value data structure. In Elixir, when we have a list of tuples and the first item of the tuple (i.e. the key) is an atom, we call it a keyword list: - -```iex -iex> list = [{:a, 1}, {:b, 2}] -[a: 1, b: 2] -iex> list == [a: 1, b: 2] -true -``` - -As you can see above, Elixir supports a special syntax for defining such lists: `[key: value]`. Underneath it maps to the same list of tuples as above. Since keyword lists are lists, we can use all operations available to lists. For example, we can use `++` to add new values to a keyword list: - -```iex -iex> list ++ [c: 3] -[a: 1, b: 2, c: 3] -iex> [a: 0] ++ list -[a: 0, a: 1, b: 2] -``` - -Note that values added to the front are the ones fetched on lookup: - -```iex -iex> new_list = [a: 0] ++ list -[a: 0, a: 1, b: 2] -iex> new_list[:a] -0 -``` - -Keyword lists are important because they have three special characteristics: - - * Keys must be atoms. - * Keys are ordered, as specified by the developer. - * Keys can be given more than once. - -For example, [the Ecto library](https://github.com/elixir-lang/ecto) makes use of these features to provide an elegant DSL for writing database queries: - -```elixir -query = from w in Weather, - where: w.prcp > 0, - where: w.temp < 20, - select: w -``` - -These characteristics are what prompted keyword lists to be the default mechanism for passing options to functions in Elixir. In chapter 5, when we discussed the `if/2` macro, we mentioned the following syntax is supported: - -```iex -iex> if false, do: :this, else: :that -:that -``` - -The `do:` and `else:` pairs are keyword lists! In fact, the call above is equivalent to: - -```iex -iex> if(false, [do: :this, else: :that]) -:that -``` - -Which, as we have seen above, is the same as: - -```iex -iex> if(false, [{:do, :this}, {:else, :that}]) -:that -``` - -In general, when the keyword list is the last argument of a function, the square brackets are optional. - -Although we can pattern match on keyword lists, it is rarely done in practice since pattern matching on lists requires the number of items and their order to match: - -```iex -iex> [a: a] = [a: 1] -[a: 1] -iex> a -1 -iex> [a: a] = [a: 1, b: 2] -** (MatchError) no match of right hand side value: [a: 1, b: 2] -iex> [b: b, a: a] = [a: 1, b: 2] -** (MatchError) no match of right hand side value: [a: 1, b: 2] -``` - -In order to manipulate keyword lists, Elixir provides [the `Keyword` module](https://hexdocs.pm/elixir/Keyword.html). Remember, though, keyword lists are simply lists, and as such they provide the same linear performance characteristics as lists. The longer the list, the longer it will take to find a key, to count the number of items, and so on. For this reason, keyword lists are used in Elixir mainly for passing optional values. If you need to store many items or guarantee one-key associates with at maximum one-value, you should use maps instead. - -## Maps - -Whenever you need a key-value store, maps are the "go to" data structure in Elixir. A map is created using the `%{}` syntax: - -```iex -iex> map = %{:a => 1, 2 => :b} -%{2 => :b, :a => 1} -iex> map[:a] -1 -iex> map[2] -:b -iex> map[:c] -nil -``` - -Compared to keyword lists, we can already see two differences: - - * Maps allow any value as a key. - * Maps' keys do not follow any ordering. - -In contrast to keyword lists, maps are very useful with pattern matching. When a map is used in a pattern, it will always match on a subset of the given value: - -```iex -iex> %{} = %{:a => 1, 2 => :b} -%{2 => :b, :a => 1} -iex> %{:a => a} = %{:a => 1, 2 => :b} -%{2 => :b, :a => 1} -iex> a -1 -iex> %{:c => c} = %{:a => 1, 2 => :b} -** (MatchError) no match of right hand side value: %{2 => :b, :a => 1} -``` - -As shown above, a map matches as long as the keys in the pattern exist in the given map. Therefore, an empty map matches all maps. - -Variables can be used when accessing, matching and adding map keys: - -```iex -iex> n = 1 -1 -iex> map = %{n => :one} -%{1 => :one} -iex> map[n] -:one -iex> %{^n => :one} = %{1 => :one, 2 => :two, 3 => :three} -%{1 => :one, 2 => :two, 3 => :three} -``` - -[The `Map` module](https://hexdocs.pm/elixir/Map.html) provides a very similar API to the `Keyword` module with convenience functions to manipulate maps: - -```iex -iex> Map.get(%{:a => 1, 2 => :b}, :a) -1 -iex> Map.to_list(%{:a => 1, 2 => :b}) -[{2, :b}, {:a, 1}] -``` - -When all the keys in a map are atoms, you can use the keyword syntax for convenience: - -```iex -iex> map = %{a: 1, b: 2} -%{a: 1, b: 2} -``` - -Another interesting property of maps is that they provide their own syntax for updating and accessing atom keys: - -```iex -iex> map = %{:a => 1, 2 => :b} -%{2 => :b, :a => 1} - -iex> map.a -1 -iex> map.c -** (KeyError) key :c not found in: %{2 => :b, :a => 1} - -iex> %{map | :a => 2} -%{2 => :b, :a => 2} -iex> %{map | :c => 3} -** (KeyError) key :c not found in: %{2 => :b, :a => 1} -``` - -Both access and update syntaxes above require the given keys to exist. For example, accessing and updating the `:c` key failed because there is no `:c` in the map. - -Elixir developers typically prefer to use the `map.field` syntax and pattern matching instead of the functions in the `Map` module when working with maps because they lead to an assertive style of programming. [This blog post](http://blog.plataformatec.com.br/2014/09/writing-assertive-code-with-elixir/) provides insight and examples on how you get more concise and faster software by writing assertive code in Elixir. - -> Note: Maps were recently introduced into the Erlang VM and only from Elixir v1.2 they are capable of holding millions of keys efficiently. Therefore, if you are working with previous Elixir versions (v1.0 or v1.1) and you need to support at least hundreds of keys, you may consider using [the `HashDict` module](https://hexdocs.pm/elixir/HashDict.html). - -## Nested data structures - -Often we will have maps inside maps, or even keywords lists inside maps, and so forth. Elixir provides conveniences for manipulating nested data structures via the `put_in/2`, `update_in/2` and other macros giving the same conveniences you would find in imperative languages while keeping the immutable properties of the language. - -Imagine you have the following structure: - -```iex -iex> users = [ - john: %{name: "John", age: 27, languages: ["Erlang", "Ruby", "Elixir"]}, - mary: %{name: "Mary", age: 29, languages: ["Elixir", "F#", "Clojure"]} -] -[john: %{age: 27, languages: ["Erlang", "Ruby", "Elixir"], name: "John"}, - mary: %{age: 29, languages: ["Elixir", "F#", "Clojure"], name: "Mary"}] -``` - -We have a keyword list of users where each value is a map containing the name, age and a list of programming languages each user likes. If we wanted to access the age for john, we could write: - -```iex -iex> users[:john].age -27 -``` - -It happens we can also use this same syntax for updating the value: - -```iex -iex> users = put_in users[:john].age, 31 -[john: %{age: 31, languages: ["Erlang", "Ruby", "Elixir"], name: "John"}, - mary: %{age: 29, languages: ["Elixir", "F#", "Clojure"], name: "Mary"}] -``` - -The `update_in/2` macro is similar but allows us to pass a function that controls how the value changes. For example, let's remove "Clojure" from Mary's list of languages: - -```iex -iex> users = update_in users[:mary].languages, &List.delete(&1, "Clojure") -[john: %{age: 31, languages: ["Erlang", "Ruby", "Elixir"], name: "John"}, - mary: %{age: 29, languages: ["Elixir", "F#"], name: "Mary"}] -``` - -There is more to learn about `put_in/2` and `update_in/2`, including the `get_and_update_in/2` that allows us to extract a value and update the data structure at once. There are also `put_in/3`, `update_in/3` and `get_and_update_in/3` which allow dynamic access into the data structure. [Check their respective documentation in the `Kernel` module for more information](https://hexdocs.pm/elixir/Kernel.html). - -This concludes our introduction to associative data structures in Elixir. You will find out that, given keyword lists and maps, you will always have the right tool to tackle problems that require associative data structures in Elixir. diff --git a/getting-started/meta/domain-specific-languages.markdown b/getting-started/meta/domain-specific-languages.markdown index 1fbd5708f..96c2c5f2b 100644 --- a/getting-started/meta/domain-specific-languages.markdown +++ b/getting-started/meta/domain-specific-languages.markdown @@ -1,206 +1,5 @@ --- -layout: getting-started -title: Domain Specific Languages +layout: redirect +sitemap: false +redirect_to: domain-specific-languages --- - -# {{ page.title }} - -{% include toc.html %} - -## Foreword - -[Domain Specific Languages (DSL)](https://en.wikipedia.org/wiki/Domain-specific_language) allow developers to tailor their application to a particular domain. You don't need macros in order to have a DSL: every data structure and every function you define in your module is part of your Domain Specific Language. - -For example, imagine we want to implement a Validator module which provides a data validation domain specific language. We could implement it using data structures, functions or macros. Let's see how those different DSLs would look like: - -```elixir -# 1. data structures -import Validator -validate user, name: [length: 1..100], - email: [matches: ~r/@/] - -# 2. functions -import Validator -user -|> validate_length(:name, 1..100) -|> validate_matches(:email, ~r/@/) - -# 3. macros + modules -defmodule MyValidator do - use Validator - validate_length :name, 1..100 - validate_matches :email, ~r/@/ -end - -MyValidator.validate(user) -``` - -Of all the approaches above, the first is definitely the most flexible. If our domain rules can be encoded with data structures, they are by far the easiest to compose and implement, as Elixir's standard library is filled with functions for manipulating different data types. - -The second approach uses function calls which better suits more complex APIs (for example, if you need to pass many options) and reads nicely in Elixir thanks to the pipe operator. - -The third approach, uses macros, and is by far the most complex. It will take more lines of code to implement, it is hard and expensive to test (compared to testing simple functions), and it limits how the user may use the library since all validations need to be defined inside a module. - -To drive the point home, imagine you want to validate a certain attribute only if a given condition is met. We could easily achieve it with the first solution, by manipulating the data structure accordingly, or with the second solution by using conditionals (if/else) before invoking the function. However it is impossible to do so with the macros approach unless its DSL is augmented. - -In other words: - - data > functions > macros - -That said, there are still cases where using macros and modules to build domain specific languages is useful. Since we have explored data structures and function definitions in the Getting Started guide, this chapter will explore how to use macros and module attributes to tackle more complex DSLs. - -## Building our own test case - -The goal in this chapter is to build a module named `TestCase` that allows us to write the following: - -```elixir -defmodule MyTest do - use TestCase - - test "arithmetic operations" do - 4 = 2 + 2 - end - - test "list operations" do - [1, 2, 3] = [1, 2] ++ [3] - end -end - -MyTest.run -``` - -In the example above, by using `TestCase`, we can write tests using the `test` macro, which defines a function named `run` to automatically run all tests for us. Our prototype will rely on the match operator (`=`) as a mechanism to do assertions. - -## The `test` macro - -Let's start by creating a module that defines and imports the `test` macro when used: - -```elixir -defmodule TestCase do - # Callback invoked by `use`. - # - # For now it returns a quoted expression that - # imports the module itself into the user code. - @doc false - defmacro __using__(_opts) do - quote do - import TestCase - end - end - - @doc """ - Defines a test case with the given description. - - ## Examples - - test "arithmetic operations" do - 4 = 2 + 2 - end - - """ - defmacro test(description, do: block) do - function_name = String.to_atom("test " <> description) - quote do - def unquote(function_name)(), do: unquote(block) - end - end -end -``` - -Assuming we defined `TestCase` in a file named `tests.exs`, we can open it up by running `iex tests.exs` and define our first tests: - -```iex -iex> defmodule MyTest do -...> use TestCase -...> -...> test "hello" do -...> "hello" = "world" -...> end -...> end -``` - -For now we don't have a mechanism to run tests, but we know that a function named "test hello" was defined behind the scenes. When we invoke it, it should fail: - -```iex -iex> MyTest."test hello"() -** (MatchError) no match of right hand side value: "world" -``` - -## Storing information with attributes - -In order to finish our `TestCase` implementation, we need to be able to access all defined test cases. One way of doing this is by retrieving the tests at runtime via `__MODULE__.__info__(:functions)`, which returns a list of all functions in a given module. However, considering that we may want to store more information about each test besides the test name, a more flexible approach is required. - -When discussing module attributes in earlier chapters, we mentioned how they can be used as temporary storage. That's exactly the property we will apply in this section. - -In the `__using__/1` implementation, we will initialize a module attribute named `@tests` to an empty list, then store the name of each defined test in this attribute so the tests can be invoked from the `run` function. - -Here is the updated code for the `TestCase` module: - -```elixir -defmodule TestCase do - @doc false - defmacro __using__(_opts) do - quote do - import TestCase - - # Initialize @tests to an empty list - @tests [] - - # Invoke TestCase.__before_compile__/1 before the module is compiled - @before_compile TestCase - end - end - - @doc """ - Defines a test case with the given description. - - ## Examples - - test "arithmetic operations" do - 4 = 2 + 2 - end - - """ - defmacro test(description, do: block) do - function_name = String.to_atom("test " <> description) - quote do - # Prepend the newly defined test to the list of tests - @tests [unquote(function_name) | @tests] - def unquote(function_name)(), do: unquote(block) - end - end - - # This will be invoked right before the target module is compiled - # giving us the perfect opportunity to inject the `run/0` function - @doc false - defmacro __before_compile__(env) do - quote do - def run do - Enum.each @tests, fn name -> - IO.puts "Running #{name}" - apply(__MODULE__, name, []) - end - end - end - end -end -``` - -By starting a new IEx session, we can now define our tests and run them: - -```iex -iex> defmodule MyTest do -...> use TestCase -...> -...> test "hello" do -...> "hello" = "world" -...> end -...> end -iex> MyTest.run -Running test hello -** (MatchError) no match of right hand side value: "world" -``` - -Although we have overlooked some details, this is the main idea behind creating domain specific modules in Elixir. Macros enable us to return quoted expressions that are executed in the caller, which we can then use to transform code and store relevant information in the target module via module attributes. Finally, callbacks such as `@before_compile` allow us to inject code into the module when its definition is complete. - -Besides `@before_compile`, there are other useful module attributes like `@on_definition` and `@after_compile`, which you can read more about in [the docs for the `Module` module](https://hexdocs.pm/elixir/Module.html). You can also find useful information about macros and the compilation environment in the documentation for the [`Macro` module](https://hexdocs.pm/elixir/Macro.html) and [`Macro.Env`](https://hexdocs.pm/elixir/Macro.Env.html). diff --git a/getting-started/meta/index.html b/getting-started/meta/index.html index e41e5b0a7..70c16019b 100644 --- a/getting-started/meta/index.html +++ b/getting-started/meta/index.html @@ -1,5 +1,5 @@ - - - - - +--- +layout: redirect +sitemap: false +redirect_to: quote-and-unquote +--- diff --git a/getting-started/meta/macros.markdown b/getting-started/meta/macros.markdown index bd7cc8843..349aee294 100644 --- a/getting-started/meta/macros.markdown +++ b/getting-started/meta/macros.markdown @@ -1,294 +1,5 @@ --- -layout: getting-started -title: Macros +layout: redirect +sitemap: false +redirect_to: macros --- - -# {{ page.title }} - -{% include toc.html %} - -## Foreword - -Even though Elixir attempts its best to provide a safe environment for macros, the major responsibility of writing clean code with macros falls on developers. Macros are harder to write than ordinary Elixir functions and it's considered to be bad style to use them when they're not necessary. So write macros responsibly. - -Elixir already provides mechanisms to write your every day code in a simple and readable fashion by using its data structures and functions. Macros should only be used as a last resort. Remember that **explicit is better than implicit**. **Clear code is better than concise code.** - -## Our first macro - -Macros in Elixir are defined via `defmacro/2`. - -> For this chapter, we will be using files instead of running code samples in IEx. That's because the code samples will span multiple lines of code and typing them all in IEx can be counter-productive. You should be able to run the code samples by saving them into a `macros.exs` file and running it with `elixir macros.exs` or `iex macros.exs`. - -In order to better understand how macros work, let's create a new module where we are going to implement `unless`, which does the opposite of `if`, as a macro and as a function: - -```elixir -defmodule Unless do - def fun_unless(clause, do: expression) do - if(!clause, do: expression) - end - - defmacro macro_unless(clause, do: expression) do - quote do - if(!unquote(clause), do: unquote(expression)) - end - end -end -``` - -The function receives the arguments and passes them to `if`. However, as we learned in the [previous chapter](/getting-started/meta/quote-and-unquote.html), the macro will receive quoted expressions, inject them into the quote, and finally return another quoted expression. - -Let's start `iex` with the module above: - -```bash -$ iex macros.exs -``` - -And play with those definitions: - -```iex -iex> require Unless -iex> Unless.macro_unless true, do: IO.puts "this should never be printed" -nil -iex> Unless.fun_unless true, do: IO.puts "this should never be printed" -"this should never be printed" -nil -``` - -Note that in our macro implementation, the sentence was not printed, although it was printed in our function implementation. That's because the arguments to a function call are evaluated before calling the function. However, macros do not evaluate their arguments. Instead, they receive the arguments as quoted expressions which are then transformed into other quoted expressions. In this case, we have rewritten our `unless` macro to become an `if` behind the scenes. - -In other words, when invoked as: - -```elixir -Unless.macro_unless true, do: IO.puts "this should never be printed" -``` - -Our `macro_unless` macro received the following: - -{% raw %} -```elixir -macro_unless(true, [do: {{:., [], [{:__aliases__, [alias: false], [:IO]}, :puts]}, [], ["this should never be printed"]}]) -``` -{% endraw %} - -And it then returned a quoted expression as follows: - -{% raw %} -```elixir -{:if, [], - [{:!, [], [true]}, - [do: {{:., [], - [{:__aliases__, - [], [:IO]}, - :puts]}, [], ["this should never be printed"]}]]} -``` -{% endraw %} - -We can actually verify that this is the case by using `Macro.expand_once/2`: - -```iex -iex> expr = quote do: Unless.macro_unless(true, do: IO.puts "this should never be printed") -iex> res = Macro.expand_once(expr, __ENV__) -iex> IO.puts Macro.to_string(res) -if(!true) do - IO.puts("this should never be printed") -end -:ok -``` - -`Macro.expand_once/2` receives a quoted expression and expands it according to the current environment. In this case, it expanded/invoked the `Unless.macro_unless/2` macro and returned its result. We then proceeded to convert the returned quoted expression to a string and print it (we will talk about `__ENV__` later in this chapter). - -That's what macros are all about. They are about receiving quoted expressions and transforming them into something else. In fact, `unless/2` in Elixir is implemented as a macro: - -```elixir -defmacro unless(clause, do: expression) do - quote do - if(!unquote(clause), do: unquote(expression)) - end -end -``` - -Constructs such as `unless/2`, `defmacro/2`, `def/2`, `defprotocol/2`, and many others used throughout this getting started guide are implemented in pure Elixir, often as a macro. This means that the constructs being used to build the language can be used by developers to extend the language to the domains they are working on. - -We can define any function and macro we want, including ones that override the built-in definitions provided by Elixir. The only exceptions are Elixir special forms which are not implemented in Elixir and therefore cannot be overridden, [the full list of special forms is available in `Kernel.SpecialForms`](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#summary). - -## Macros hygiene - -Elixir macros have late resolution. This guarantees that a variable defined inside a quote won't conflict with a variable defined in the context where that macro is expanded. For example: - -```elixir -defmodule Hygiene do - defmacro no_interference do - quote do: a = 1 - end -end - -defmodule HygieneTest do - def go do - require Hygiene - a = 13 - Hygiene.no_interference - a - end -end - -HygieneTest.go -# => 13 -``` - -In the example above, even though the macro injects `a = 1`, it does not affect the variable `a` defined by the `go` function. If a macro wants to explicitly affect the context, it can use `var!`: - -```elixir -defmodule Hygiene do - defmacro interference do - quote do: var!(a) = 1 - end -end - -defmodule HygieneTest do - def go do - require Hygiene - a = 13 - Hygiene.interference - a - end -end - -HygieneTest.go -# => 1 -``` - -Variable hygiene only works because Elixir annotates variables with their context. For example, a variable `x` defined on line 3 of a module would be represented as: - -```elixir -{:x, [line: 3], nil} -``` - -However, a quoted variable is represented as: - -```elixir -defmodule Sample do - def quoted do - quote do: x - end -end - -Sample.quoted #=> {:x, [line: 3], Sample} -``` - -Notice that the third element in the quoted variable is the atom `Sample`, instead of `nil`, which marks the variable as coming from the `Sample` module. Therefore, Elixir considers these two variables as coming from different contexts and handles them accordingly. - -Elixir provides similar mechanisms for imports and aliases too. This guarantees that a macro will behave as specified by its source module rather than conflicting with the target module where the macro is expanded. Hygiene can be bypassed under specific situations by using macros like `var!/2` and `alias!/2`, although one must be careful when using those as they directly change the user environment. - -Sometimes variable names might be dynamically created. In such cases, `Macro.var/2` can be used to define new variables: - -```elixir -defmodule Sample do - defmacro initialize_to_char_count(variables) do - Enum.map variables, fn(name) -> - var = Macro.var(name, nil) - length = name |> Atom.to_string |> String.length - quote do - unquote(var) = unquote(length) - end - end - end - - def run do - initialize_to_char_count [:red, :green, :yellow] - [red, green, yellow] - end -end - -> Sample.run #=> [3, 5, 6] -``` - -Take note of the second argument to `Macro.var/2`. This is the context being used and will determine hygiene as described in the next section. - -## The environment - -When calling `Macro.expand_once/2` earlier in this chapter, we used the special form `__ENV__`. - -`__ENV__` returns an instance of the `Macro.Env` struct which contains useful information about the compilation environment, including the current module, file and line, all variables defined in the current scope, as well as imports, requires and so on: - -```iex -iex> __ENV__.module -nil -iex> __ENV__.file -"iex" -iex> __ENV__.requires -[IEx.Helpers, Kernel, Kernel.Typespec] -iex> require Integer -nil -iex> __ENV__.requires -[IEx.Helpers, Integer, Kernel, Kernel.Typespec] -``` - -Many of the functions in the `Macro` module expect an environment. You can read more about these functions in [the docs for the `Macro` module](https://hexdocs.pm/elixir/Macro.html) and learn more about the compilation environment in the [docs for `Macro.Env`](https://hexdocs.pm/elixir/Macro.Env.html). - -## Private macros - -Elixir also supports private macros via `defmacrop`. As private functions, these macros are only available inside the module that defines them, and only at compilation time. - -It is important that a macro is defined before its usage. Failing to define a macro before its invocation will raise an error at runtime, since the macro won't be expanded and will be translated to a function call: - -```iex -iex> defmodule Sample do -...> def four, do: two + two -...> defmacrop two, do: 2 -...> end -** (CompileError) iex:2: function two/0 undefined -``` - -## Write macros responsibly - -Macros are a powerful construct and Elixir provides many mechanisms to ensure they are used responsibly. - -* Macros are hygienic: by default, variables defined inside a macro are not going to affect the user code. Furthermore, function calls and aliases available in the macro context are not going to leak into the user context. - -* Macros are lexical: it is impossible to inject code or macros globally. In order to use a macro, you need to explicitly `require` or `import` the module that defines the macro. - -* Macros are explicit: it is impossible to run a macro without explicitly invoking it. For example, some languages allow developers to completely rewrite functions behind the scenes, often via parse transforms or via some reflection mechanisms. In Elixir, a macro must be explicitly invoked in the caller during compilation time. - -* Macros' language is clear: many languages provide syntax shortcuts for `quote` and `unquote`. In Elixir, we preferred to have them explicitly spelled out, in order to clearly delimit the boundaries of a macro definition and its quoted expressions. - -Even with such guarantees, the developer plays a big role when writing macros responsibly. If you are confident you need to resort to macros, remember that macros are not your API. Keep your macro definitions short, including their quoted contents. For example, instead of writing a macro like this: - -```elixir -defmodule MyModule do - defmacro my_macro(a, b, c) do - quote do - do_this(unquote(a)) - ... - do_that(unquote(b)) - ... - and_that(unquote(c)) - end - end -end -``` - -write: - -```elixir -defmodule MyModule do - defmacro my_macro(a, b, c) do - quote do - # Keep what you need to do here to a minimum - # and move everything else to a function - do_this_that_and_that(unquote(a), unquote(b), unquote(c)) - end - end - - def do_this_that_and_that(a, b, c) do - do_this(a) - ... - do_that(b) - ... - and_that(c) - end -end -``` - -This makes your code clearer and easier to test and maintain, as you can invoke and test `do_this_that_and_that/3` directly. It also helps you design an actual API for developers that do not want to rely on macros. - -With those lessons, we finish our introduction to macros. The next chapter is a brief discussion on DSLs that shows how we can mix macros and module attributes to annotate and extend modules and functions. diff --git a/getting-started/meta/quote-and-unquote.markdown b/getting-started/meta/quote-and-unquote.markdown index 28ec811ae..70c16019b 100644 --- a/getting-started/meta/quote-and-unquote.markdown +++ b/getting-started/meta/quote-and-unquote.markdown @@ -1,155 +1,5 @@ --- -layout: getting-started -title: Quote and unquote +layout: redirect +sitemap: false +redirect_to: quote-and-unquote --- - -# {{ page.title }} - -{% include toc.html %} - -An Elixir program can be represented by its own data structures. In this chapter, we will learn what those structures look like and how to compose them. The concepts learned in this chapter are the building blocks for macros, which we are going to take a deeper look at in the next chapter. - -## Quoting - -The building block of an Elixir program is a tuple with three elements. For example, the function call `sum(1, 2, 3)` is represented internally as: - -```elixir -{:sum, [], [1, 2, 3]} -``` - -You can get the representation of any expression by using the `quote` macro: - -```iex -iex> quote do: sum(1, 2, 3) -{:sum, [], [1, 2, 3]} -``` - -The first element is the function name, the second is a keyword list containing metadata and the third is the arguments list. - -Operators are also represented as such tuples: - -```iex -iex> quote do: 1 + 2 -{:+, [context: Elixir, import: Kernel], [1, 2]} -``` - -Even a map is represented as a call to `%{}`: - -```iex -iex> quote do: %{1 => 2} -{:%{}, [], [{1, 2}]} -``` - -Variables are also represented using such triplets, except the last element is an atom, instead of a list: - -```iex -iex> quote do: x -{:x, [], Elixir} -``` - -When quoting more complex expressions, we can see that the code is represented in such tuples, which are often nested inside each other in a structure resembling a tree. Many languages would call such representations an Abstract Syntax Tree (AST). Elixir calls them quoted expressions: - -```iex -iex> quote do: sum(1, 2 + 3, 4) -{:sum, [], [1, {:+, [context: Elixir, import: Kernel], [2, 3]}, 4]} -``` - -Sometimes when working with quoted expressions, it may be useful to get the textual code representation back. This can be done with `Macro.to_string/1`: - -```iex -iex> Macro.to_string(quote do: sum(1, 2 + 3, 4)) -"sum(1, 2 + 3, 4)" -``` - -In general, the tuples above are structured according to the following format: - -```elixir -{atom | tuple, list, list | atom} -``` - -* The first element is an atom or another tuple in the same representation; -* The second element is a keyword list containing metadata, like numbers and contexts; -* The third element is either a list of arguments for the function call or an atom. When this element is an atom, it means the tuple represents a variable. - -Besides the tuple defined above, there are five Elixir literals that, when quoted, return themselves (and not a tuple). They are: - -```elixir -:sum #=> Atoms -1.0 #=> Numbers -[1, 2] #=> Lists -"strings" #=> Strings -{key, value} #=> Tuples with two elements -``` - -Most Elixir code has a straight-forward translation to its underlying quoted expression. We recommend you try out different code samples and see what the results are. For example, what does `String.upcase("foo")` expand to? We have also learned that `if(true, do: :this, else: :that)` is the same as `if true do :this else :that end`. How does this affirmation hold with quoted expressions? - -## Unquoting - -Quote is about retrieving the inner representation of some particular chunk of code. However, sometimes it may be necessary to inject some other particular chunk of code inside the representation we want to retrieve. - -For example, imagine you have a variable `number` which contains the number you want to inject inside a quoted expression. - -```iex -iex> number = 13 -iex> Macro.to_string(quote do: 11 + number) -"11 + number" -``` - -That's not what we wanted, since the value of the `number` variable has not been injected and `number` has been quoted in the expression. In order to inject the *value* of the `number` variable, `unquote` has to be used inside the quoted representation: - -```iex -iex> number = 13 -iex> Macro.to_string(quote do: 11 + unquote(number)) -"11 + 13" -``` - -`unquote` can even be used to inject function names: - -```iex -iex> fun = :hello -iex> Macro.to_string(quote do: unquote(fun)(:world)) -"hello(:world)" -``` - -In some cases, it may be necessary to inject many values inside a list. For example, imagine you have a list containing `[1, 2, 6]` and we want to inject `[3, 4, 5]` into it. Using `unquote` won't yield the desired result: - -```iex -iex> inner = [3, 4, 5] -iex> Macro.to_string(quote do: [1, 2, unquote(inner), 6]) -"[1, 2, [3, 4, 5], 6]" -``` - -That's when `unquote_splicing` becomes handy: - -```iex -iex> inner = [3, 4, 5] -iex> Macro.to_string(quote do: [1, 2, unquote_splicing(inner), 6]) -"[1, 2, 3, 4, 5, 6]" -``` - -Unquoting is very useful when working with macros. When writing macros, developers are able to receive code chunks and inject them inside other code chunks, which can be used to transform code or write code that generates code during compilation. - -## Escaping - -As we saw at the beginning of this chapter, only some values are valid quoted expressions in Elixir. For example, a map is not a valid quoted expression. Neither is a tuple with four elements. However, such values *can* be expressed as a quoted expression: - -```iex -iex> quote do: %{1 => 2} -{:%{}, [], [{1, 2}]} -``` - -In some cases, you may need to inject such *values* into *quoted expressions*. To do that, we need to first escape those values into quoted expressions with the help of `Macro.escape/1`: - -```iex -iex> map = %{hello: :world} -iex> Macro.escape(map) -{:%{}, [], [hello: :world]} -``` - -Macros receive quoted expressions and must return quoted expressions. However, sometimes during the execution of a macro, you may need to work with values and making a distinction between values and quoted expressions will be required. - -In other words, it is important to make a distinction between a regular Elixir value (like a list, a map, a process, a reference, etc) and a quoted expression. Some values, such as integers, atoms and strings, have a quoted expression equal to the value itself. Other values, like maps, need to be explicitly converted. Finally, values like functions and references cannot be converted to a quoted expression at all. - -You can read more about `quote` and `unquote` in the [`Kernel.SpecialForms` module](https://hexdocs.pm/elixir/Kernel.SpecialForms.html). Documentation for `Macro.escape/1` and other functions related to quoted expressions can be found in the [`Macro` module](https://hexdocs.pm/elixir/Macro.html). - -In this introduction we have laid the groundwork to finally write our first macro, so let's move to the next chapter. diff --git a/getting-started/mix-otp/agent.markdown b/getting-started/mix-otp/agent.markdown index dd556bb55..419cc43f7 100644 --- a/getting-started/mix-otp/agent.markdown +++ b/getting-started/mix-otp/agent.markdown @@ -1,189 +1,5 @@ --- -layout: getting-started -title: Agent +layout: redirect +sitemap: false +redirect_to: agents --- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -In this chapter, we will create a module named `KV.Bucket`. This module will be responsible for storing our key-value entries in a way that allows them to be read and modified by other processes. - -If you have skipped the Getting Started guide or read it long ago, be sure to re-read the [Processes](/getting-started/processes.html) chapter. We will use it as a starting point. - -## The trouble with state - -Elixir is an immutable language where nothing is shared by default. If we want to provide state, where we create buckets putting and reading values from multiple places, we have two main options in Elixir: - -* Processes -* [ETS (Erlang Term Storage)](http://www.erlang.org/doc/man/ets.html) - -We have already talked about processes, while ETS is something new that we will explore later in this guide. When it comes to processes though, we rarely hand-roll our own, instead we use the abstractions available in Elixir and OTP: - -* [Agent](https://hexdocs.pm/elixir/Agent.html) - Simple wrappers around state. -* [GenServer](https://hexdocs.pm/elixir/GenServer.html) - "Generic servers" (processes) that encapsulate state, provide sync and async calls, support code reloading, and more. -* [GenEvent](https://hexdocs.pm/elixir/GenEvent.html) - "Generic event" managers that allow publishing events to multiple handlers. -* [Task](https://hexdocs.pm/elixir/Task.html) - Asynchronous units of computation that allow spawning a process and potentially retrieving its result at a later time. - -We will explore most of these abstractions in this guide. Keep in mind that they are all implemented on top of processes using the basic features provided by the VM, like `send`, `receive`, `spawn` and `link`. - -## Agents - -[Agents](https://hexdocs.pm/elixir/Agent.html) are simple wrappers around state. If all you want from a process is to keep state, agents are a great fit. Let's start an `iex` session inside the project with: - -```bash -$ iex -S mix -``` - -And play a bit with agents: - -```iex -iex> {:ok, agent} = Agent.start_link fn -> [] end -{:ok, #PID<0.57.0>} -iex> Agent.update(agent, fn list -> ["eggs" | list] end) -:ok -iex> Agent.get(agent, fn list -> list end) -["eggs"] -iex> Agent.stop(agent) -:ok -``` - -We started an agent with an initial state of an empty list. We updated the agent's state, adding our new item to the head of the list. The second argument of [`Agent.update/3`](https://hexdocs.pm/elixir/Agent.html#update/3) is a function that takes the agent's current state as input and returns its desired new state. Finally, we retrieved the whole list. The second argument of [`Agent.get/3`](https://hexdocs.pm/elixir/Agent.html#get/3) is a function that takes the state as input and returns the value that [`Agent.get/3`](https://hexdocs.pm/elixir/Agent.html#get/3) itself will return. Once we are done with the agent, we can call [`Agent.stop/3`](https://hexdocs.pm/elixir/Agent.html#stop/3) to terminate the agent process. - -Let's implement our `KV.Bucket` using agents. But before starting the implementation, let's first write some tests. Create a file at `test/kv/bucket_test.exs` (remember the `.exs` extension) with the following: - -```elixir -defmodule KV.BucketTest do - use ExUnit.Case, async: true - - test "stores values by key" do - {:ok, bucket} = KV.Bucket.start_link - assert KV.Bucket.get(bucket, "milk") == nil - - KV.Bucket.put(bucket, "milk", 3) - assert KV.Bucket.get(bucket, "milk") == 3 - end -end -``` - -Our first test starts a new `KV.Bucket` and performs some `get/2` and `put/3` operations on it, asserting the result. We don't need to explicitly stop the agent because it is linked to the test process and the agent is shut down automatically once the test finishes. This will always work unless the process is named. - -Also note the `async: true` option passed to `ExUnit.Case`. This option makes the test case run in parallel with other `:async` test cases by using multiple cores in our machine. This is extremely useful to speed up our test suite. However, `:async` must *only* be set if the test case does not rely on or change any global values. For example, if the test requires writing to the filesystem, registering processes, or accessing a database, keep it synchronous (omit the `:async` option) to avoid race conditions between tests. - -Async or not, our new test should obviously fail, as none of the functionality is implemented in the module being tested. - -In order to fix the failing test, let's create a file at `lib/kv/bucket.ex` with the contents below. Feel free to give a try at implementing the `KV.Bucket` module yourself using agents before peeking at the implementation below. - -```elixir -defmodule KV.Bucket do - @doc """ - Starts a new bucket. - """ - def start_link do - Agent.start_link(fn -> %{} end) - end - - @doc """ - Gets a value from the `bucket` by `key`. - """ - def get(bucket, key) do - Agent.get(bucket, &Map.get(&1, key)) - end - - @doc """ - Puts the `value` for the given `key` in the `bucket`. - """ - def put(bucket, key, value) do - Agent.update(bucket, &Map.put(&1, key, value)) - end -end -``` - -We are using a map to store our keys and values. The capture operator, `&`, is introduced in [the Getting Started guide](/getting-started/modules.html#function-capturing). - -Now that the `KV.Bucket` module has been defined, our test should pass! You can try it yourself by running: `mix test`. - -## Test setup with ExUnit callbacks - -Before moving on and adding more features to `KV.Bucket`, let's talk about ExUnit callbacks. As you may expect, all `KV.Bucket` tests will require a bucket to be started during setup and stopped after the test. Luckily, ExUnit supports callbacks that allow us to skip such repetitive tasks. - -Let's rewrite the test case to use callbacks: - -```elixir -defmodule KV.BucketTest do - use ExUnit.Case, async: true - - setup do - {:ok, bucket} = KV.Bucket.start_link - {:ok, bucket: bucket} - end - - test "stores values by key", %{bucket: bucket} do - assert KV.Bucket.get(bucket, "milk") == nil - - KV.Bucket.put(bucket, "milk", 3) - assert KV.Bucket.get(bucket, "milk") == 3 - end -end -``` - -We have first defined a setup callback with the help of the `setup/1` macro. The `setup/1` callback runs before every test, in the same process as the test itself. - -Note that we need a mechanism to pass the `bucket` pid from the callback to the test. We do so by using the *test context*. When we return `{:ok, bucket: bucket}` from the callback, ExUnit will merge the second element of the tuple (a dictionary) into the test context. The test context is a map which we can then match in the test definition, providing access to these values inside the block: - -```elixir -test "stores values by key", %{bucket: bucket} do - # `bucket` is now the bucket from the setup block -end -``` - -You can read more about ExUnit cases in the [`ExUnit.Case` module documentation](https://hexdocs.pm/ex_unit/ExUnit.Case.html) and more about callbacks in [`ExUnit.Callbacks` docs](https://hexdocs.pm/ex_unit/ExUnit.Callbacks.html). - -## Other agent actions - -Besides getting a value and updating the agent state, agents allow us to get a value and update the agent state in one function call via `Agent.get_and_update/2`. Let's implement a `KV.Bucket.delete/2` function that deletes a key from the bucket, returning its current value: - -```elixir -@doc """ -Deletes `key` from `bucket`. - -Returns the current value of `key`, if `key` exists. -""" -def delete(bucket, key) do - Agent.get_and_update(bucket, &Map.pop(&1, key)) -end -``` - -Now it is your turn to write a test for the functionality above! Also, be sure to explore [the documentation for the `Agent` module](https://hexdocs.pm/elixir/Agent.html) to learn more about them. - -## Client/Server in agents - -Before we move on to the next chapter, let's discuss the client/server dichotomy in agents. Let's expand the `delete/2` function we have just implemented: - -```elixir -def delete(bucket, key) do - Agent.get_and_update(bucket, fn dict-> - Map.pop(dict, key) - end) -end -``` - -Everything that is inside the function we passed to the agent happens in the agent process. In this case, since the agent process is the one receiving and responding to our messages, we say the agent process is the server. Everything outside the function is happening in the client. - -This distinction is important. If there are expensive actions to be done, you must consider if it will be better to perform these actions on the client or on the server. For example: - -```elixir -def delete(bucket, key) do - Process.sleep(1000) # puts client to sleep - Agent.get_and_update(bucket, fn dict -> - Process.sleep(1000) # puts server to sleep - Map.pop(dict, key) - end) -end -``` - -When a long action is performed on the server, all other requests to that particular server will wait until the action is done, which may cause some clients to timeout. - -In the next chapter we will explore GenServers, where the segregation between clients and servers is made even more apparent. diff --git a/getting-started/mix-otp/config-and-releases.markdown b/getting-started/mix-otp/config-and-releases.markdown new file mode 100644 index 000000000..6a141bef9 --- /dev/null +++ b/getting-started/mix-otp/config-and-releases.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: config-and-releases +--- diff --git a/getting-started/mix-otp/dependencies-and-umbrella-apps.markdown b/getting-started/mix-otp/dependencies-and-umbrella-apps.markdown index edf0e8e45..946955ce2 100644 --- a/getting-started/mix-otp/dependencies-and-umbrella-apps.markdown +++ b/getting-started/mix-otp/dependencies-and-umbrella-apps.markdown @@ -1,290 +1,5 @@ --- -layout: getting-started -title: Dependencies and umbrella projects +layout: redirect +sitemap: false +redirect_to: dependencies-and-umbrella-projects --- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -In this chapter, we will discuss how to manage dependencies in Mix. - -Our `kv` application is complete, so it's time to implement the server that will handle the requests we defined in the first chapter: - -``` -CREATE shopping -OK - -PUT shopping milk 1 -OK - -PUT shopping eggs 3 -OK - -GET shopping milk -1 -OK - -DELETE shopping eggs -OK -``` - -However, instead of adding more code to the `kv` application, we are going to build the TCP server as another application that is a client of the `kv` application. Since the whole runtime and Elixir ecosystem are geared towards applications, it makes sense to break our projects into smaller applications that work together rather than building a big, monolithic app. - -Before creating our new application, we must discuss how Mix handles dependencies. In practice, there are two kinds of dependencies we usually work with: internal and external dependencies. Mix supports mechanisms to work with both of them. - -## External dependencies - -External dependencies are the ones not tied to your business domain. For example, if you need a HTTP API for your distributed KV application, you can use the [Plug](https://github.com/elixir-lang/plug) project as an external dependency. - -Installing external dependencies is simple. Most commonly, we use the [Hex Package Manager](https://hex.pm), by listing the dependency inside the deps function in our `mix.exs` file: - -```elixir -def deps do - [{:plug, "~> 1.0"}] -end -``` - -This dependency refers to the latest version of Plug in the 1.x.x version series that has been pushed to Hex. This is indicated by the `~>` preceding the version number. For more information on specifying version requirements, see the [documentation for the Version module](https://hexdocs.pm/elixir/Version.html). - -Typically, stable releases are pushed to Hex. If you want to depend on an external dependency still in development, Mix is able to manage git dependencies too: - -```elixir -def deps do - [{:plug, git: "git://github.com/elixir-lang/plug.git"}] -end -``` - -You will notice that when you add a dependency to your project, Mix generates a `mix.lock` file that guarantees *repeatable builds*. The lock file must be checked in to your version control system, to guarantee that everyone who uses the project will use the same dependency versions as you. - -Mix provides many tasks for working with dependencies, which can be seen in `mix help`: - -```bash -$ mix help -mix deps # Lists dependencies and their status -mix deps.clean # Deletes the given dependencies' files -mix deps.compile # Compiles dependencies -mix deps.get # Gets all out of date dependencies -mix deps.tree # Prints the dependency tree -mix deps.unlock # Unlocks the given dependencies -mix deps.update # Updates the given dependencies -``` - -The most common tasks are `mix deps.get` and `mix deps.update`. Once fetched, dependencies are automatically compiled for you. You can read more about deps by typing `mix help deps`, and in the [documentation for the Mix.Tasks.Deps module](https://hexdocs.pm/mix/Mix.Tasks.Deps.html). - -## Internal dependencies - -Internal dependencies are the ones that are specific to your project. They usually don't make sense outside the scope of your project/company/organization. Most of the time, you want to keep them private, whether due to technical, economic or business reasons. - -If you have an internal dependency, Mix supports two methods to work with them: git repositories or umbrella projects. - -For example, if you push the `kv` project to a git repository, you'll need to list it in your deps code in order to use it: - -```elixir -def deps do - [{:kv, git: "https://github.com/YOUR_ACCOUNT/kv.git"}] -end -``` - -If the repository is private though, you may need to specify the private URL `git@github.com:YOUR_ACCOUNT/kv.git`. In any case, Mix will be able to fetch it for you as long as you have the proper credentials. - -Using git dependencies for internal dependencies is somewhat discouraged in Elixir. Remember that the runtime and the Elixir ecosystem already provide the concept of applications. As such, we expect you to frequently break your code into applications that can be organized logically, even within a single project. - -However, if you push every application as a separate project to a git repository, your projects may become very hard to maintain as you will spend a lot of time managing those git repositories rather than writing your code. - -For this reason, Mix supports "umbrella projects." Umbrella projects allow you to create one project that hosts many applications while keeping all of them in a single source code repository. That is exactly the style we are going to explore in the next sections. - -Let's create a new Mix project. We are going to creatively name it `kv_umbrella`, and this new project will have both the existing `kv` application and the new `kv_server` application inside. The directory structure will look like this: - - + kv_umbrella - + apps - + kv - + kv_server - -The interesting thing about this approach is that Mix has many conveniences for working with such projects, such as the ability to compile and test all applications inside `apps` with a single command. However, even though they are all listed together inside `apps`, they are still decoupled from each other, so you can build, test and deploy each application in isolation if you want to. - -So let's get started! - -## Umbrella projects - -Let's start a new project using `mix new`. This new project will be named `kv_umbrella` and we need to pass the `--umbrella` option when creating it. Do not create this new project inside the existing `kv` project! - -```bash -$ mix new kv_umbrella --umbrella -* creating .gitignore -* creating README.md -* creating mix.exs -* creating apps -* creating config -* creating config/config.exs -``` - -From the printed information, we can see far fewer files are generated. The generated `mix.exs` file is different too. Let's take a look (comments have been removed): - -```elixir -defmodule KvUmbrella.Mixfile do - use Mix.Project - - def project do - [apps_path: "apps", - build_embedded: Mix.env == :prod, - start_permanent: Mix.env == :prod, - deps: deps] - end - - defp deps do - [] - end -end -``` - -What makes this project different from the previous one is the `apps_path: "apps"` entry in the project definition. This means this project will act as an umbrella. Such projects do not have source files nor tests, although they can have their own dependencies. Each child application must be defined inside the `apps` directory. - -Let's move inside the apps directory and start building `kv_server`. This time, we are going to pass the `--sup` flag, which will tell Mix to generate a supervision tree automatically for us, instead of building one manually as we did in previous chapters: - -```bash -$ cd kv_umbrella/apps -$ mix new kv_server --module KVServer --sup -``` - -The generated files are similar to the ones we first generated for `kv`, with a few differences. Let's open up `mix.exs`: - -```elixir -defmodule KVServer.Mixfile do - use Mix.Project - - def project do - [app: :kv_server, - version: "0.1.0", - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.3", - build_embedded: Mix.env == :prod, - start_permanent: Mix.env == :prod, - deps: deps] - end - - def application do - [applications: [:logger], - mod: {KVServer, []}] - end - - defp deps do - [] - end -end - -``` - -First of all, since we generated this project inside `kv_umbrella/apps`, Mix automatically detected the umbrella structure and added four lines to the project definition: - -```elixir -build_path: "../../_build", -config_path: "../../config/config.exs", -deps_path: "../../deps", -lockfile: "../../mix.lock", -``` - -Those options mean all dependencies will be checked out to `kv_umbrella/deps`, and they will share the same build, config and lock files. This ensures dependencies will be fetched and compiled once for the whole umbrella structure, instead of once per umbrella application. - -The second change is in the `application` function inside `mix.exs`: - -```elixir -def application do - [applications: [:logger], - mod: {KVServer, []}] -end -``` - -Because we passed the `--sup` flag, Mix automatically added `mod: {KVServer, []}`, specifying that `KVServer` is our application callback module. `KVServer` will start our application supervision tree. - -In fact, let's open up `lib/kv_server.ex`: - -```elixir -defmodule KVServer do - use Application - - def start(_type, _args) do - import Supervisor.Spec, warn: false - - children = [ - # worker(KVServer.Worker, [arg1, arg2, arg3]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) - end -end -``` - -Notice that it defines the application callback function, `start/2`, and instead of defining a supervisor named `KVServer.Supervisor` that uses the `Supervisor` module, it conveniently defined the supervisor inline! You can read more about such supervisors by reading [the Supervisor module documentation](https://hexdocs.pm/elixir/Supervisor.html). - -We can already try out our first umbrella child. We could run tests inside the `apps/kv_server` directory, but that wouldn't be much fun. Instead, go to the root of the umbrella project and run `mix test`: - -```bash -$ mix test -``` - -And it works! - -Since we want `kv_server` to eventually use the functionality we defined in `kv`, we need to add `kv` as a dependency to our application. - -## In umbrella dependencies - -Mix supports an easy mechanism to make one umbrella child depend on another. Open up `apps/kv_server/mix.exs` and change the `deps/0` function to the following: - -```elixir -defp deps do - [{:kv, in_umbrella: true}] -end -``` - -The line above makes `:kv` available as a dependency inside `:kv_server`. We can invoke the modules defined in `:kv` but it does not automatically start the `:kv` application. For that, we also need to list `:kv` as an application inside `application/0`: - -```elixir -def application do - [applications: [:logger, :kv], - mod: {KVServer, []}] -end -``` - -Now Mix will guarantee the `:kv` application is started before `:kv_server` is started. - -Finally, copy the `kv` application we have built so far to the `apps` directory in our new umbrella project. The final directory structure should match the structure we mentioned earlier: - - + kv_umbrella - + apps - + kv - + kv_server - -We now need to modify `apps/kv/mix.exs` to contain the umbrella entries we have seen in `apps/kv_server/mix.exs`. Open up `apps/kv/mix.exs` and add to the `project` function: - -```elixir -build_path: "../../_build", -config_path: "../../config/config.exs", -deps_path: "../../deps", -lockfile: "../../mix.lock", -``` - -Now you can run tests for both projects from the umbrella root with `mix test`. Sweet! - -Remember that umbrella projects are a convenience to help you organize and manage your applications. Applications inside the `apps` directory are still decoupled from each other. Dependencies between them must be explicitly listed. This allows them to be developed together, but compiled, tested and deployed independently if desired. - -## Summing up - -In this chapter we have learned more about Mix dependencies and umbrella projects. We have decided to build an umbrella project because we consider `kv` and `kv_server` to be internal dependencies that matter only in the context of this project. - -In the future, you are going to write applications and you will notice they can be extracted into a concise unit that can be used by different projects. In such cases, using Git or Hex dependencies is the way to go. - -Here are a couple questions you can ask yourself when working with dependencies. Start with: does this application make sense outside this project? - -* If no, use an umbrella project with umbrella children. -* If yes, can this project be shared outside your company / organization? - * If no, use a private git repository. - * If yes, push your code to a git repository and do frequent releases using [Hex](https://hex.pm). - -With our umbrella project up and running, it is time to start writing our server. diff --git a/getting-started/mix-otp/dependencies-and-umbrella-projects.markdown b/getting-started/mix-otp/dependencies-and-umbrella-projects.markdown new file mode 100644 index 000000000..946955ce2 --- /dev/null +++ b/getting-started/mix-otp/dependencies-and-umbrella-projects.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: dependencies-and-umbrella-projects +--- diff --git a/getting-started/mix-otp/distributed-tasks-and-configuration.markdown b/getting-started/mix-otp/distributed-tasks-and-configuration.markdown deleted file mode 100644 index 6cbf22b32..000000000 --- a/getting-started/mix-otp/distributed-tasks-and-configuration.markdown +++ /dev/null @@ -1,380 +0,0 @@ ---- -layout: getting-started -title: Distributed tasks and configuration ---- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -In this last chapter, we will go back to the `:kv` application and add a routing layer that will allow us to distribute requests between nodes based on the bucket name. - -The routing layer will receive a routing table of the following format: - -```elixir -[{?a..?m, :"foo@computer-name"}, - {?n..?z, :"bar@computer-name"}] -``` - -The router will check the first byte of the bucket name against the table and dispatch to the appropriate node based on that. For example, a bucket starting with the letter "a" (`?a` represents the Unicode codepoint of the letter "a") will be dispatched to node `foo@computer-name`. - -If the matching entry points to the node evaluating the request, then we've finished routing, and this node will perform the requested operation. If the matching entry points to a different node, we'll pass the request to this node, which will look at its own routing table (which may be different from the one in the first node) and act accordingly. If no entry matches, an error will be raised. - -You may wonder why we don't tell the node we found in our routing table to perform the requested operation directly, but instead pass the routing request on to that node to process. While a routing table as simple as the one above might reasonably be shared between all nodes, passing on the routing request in this way makes it much simpler to break the routing table into smaller pieces as our application grows. Perhaps at some point, `foo@computer-name` will only be responsible for routing bucket requests, and the buckets it handles will be dispatched to different nodes. In this way, `bar@computer-name` does not need to know anything about this change. - -> Note: we will be using two nodes in the same machine throughout this chapter. You are free to use two (or more) different machines in the same network but you need to do some prep work. First of all, you need to ensure all machines have a `~/.erlang.cookie` file with exactly the same value. Second, you need to guarantee [epmd](http://www.erlang.org/doc/man/epmd.html) is running on a port that is not blocked (you can run `epmd -d` for debug info). Third, if you want to learn more about distribution in general, we recommend [this great Distribunomicon chapter from Learn You Some Erlang](http://learnyousomeerlang.com/distribunomicon). - -## Our first distributed code - -Elixir ships with facilities to connect nodes and exchange information between them. In fact, we use the same concepts of processes, message passing and receiving messages when working in a distributed environment because Elixir processes are *location transparent*. This means that when sending a message, it doesn't matter if the recipient process is on the same node or on another node, the VM will be able to deliver the message in both cases. - -In order to run distributed code, we need to start the VM with a name. The name can be short (when in the same network) or long (requires the full computer address). Let's start a new IEx session: - -```bash -$ iex --sname foo -``` - -You can see now the prompt is slightly different and shows the node name followed by the computer name: - - Interactive Elixir - press Ctrl+C to exit (type h() ENTER for help) - iex(foo@jv)1> - -My computer is named `jv`, so I see `foo@jv` in the example above, but you will get a different result. We will use `foo@computer-name` in the following examples and you should update them accordingly when trying out the code. - -Let's define a module named `Hello` in this shell: - -```iex -iex> defmodule Hello do -...> def world, do: IO.puts "hello world" -...> end -``` - -If you have another computer on the same network with both Erlang and Elixir installed, you can start another shell on it. If you don't, you can start another IEx session in another terminal. In either case, give it the short name of `bar`: - -```bash -$ iex --sname bar -``` - -Note that inside this new IEx session, we cannot access `Hello.world/0`: - -```iex -iex> Hello.world -** (UndefinedFunctionError) undefined function: Hello.world/0 - Hello.world() -``` - -However we can spawn a new process on `foo@computer-name` from `bar@computer-name`! Let's give it a try (where `@computer-name` is the one you see locally): - -```iex -iex> Node.spawn_link :"foo@computer-name", fn -> Hello.world end -#PID<9014.59.0> -hello world -``` - -Elixir spawned a process on another node and returned its pid. The code then executed on the other node where the `Hello.world/0` function exists and invoked that function. Note that the result of "hello world" was printed on the current node `bar` and not on `foo`. In other words, the message to be printed was sent back from `foo` to `bar`. This happens because the process spawned on the other node (`foo`) still has the group leader of the current node (`bar`). We have briefly talked about group leaders in the [IO chapter](/getting-started/io-and-the-file-system.html#processes-and-group-leaders). - -We can send and receive message from the pid returned by `Node.spawn_link/2` as usual. Let's try a quick ping-pong example: - -```iex -iex> pid = Node.spawn_link :"foo@computer-name", fn -> -...> receive do -...> {:ping, client} -> send client, :pong -...> end -...> end -#PID<9014.59.0> -iex> send pid, {:ping, self} -{:ping, #PID<0.73.0>} -iex> flush -:pong -:ok -``` - -From our quick exploration, we could conclude that we should use `Node.spawn_link/2` to spawn processes on a remote node every time we need to do a distributed computation. However we have learned throughout this guide that spawning processes outside of supervision trees should be avoided if possible, so we need to look for other options. - -There are three better alternatives to `Node.spawn_link/2` that we could use in our implementation: - -1. We could use Erlang's [:rpc](http://www.erlang.org/doc/man/rpc.html) module to execute functions on a remote node. Inside the `bar@computer-name` shell above, you can call `:rpc.call(:"foo@computer-name", Hello, :world, [])` and it will print "hello world" - -2. We could have a server running on the other node and send requests to that node via the [GenServer](https://hexdocs.pm/elixir/GenServer.html) API. For example, you can call a server on a remote node by using `GenServer.call({name, node}, arg)` or passing the remote process PID as first argument - -3. We could use [tasks](https://hexdocs.pm/elixir/Task.html), which we have learned about in [a previous chapter](/getting-started/mix-otp/task-and-gen-tcp.html), as they can be spawned on both local and remote nodes - -The options above have different properties. Both `:rpc` and using a GenServer would serialize your requests on a single server, while tasks are effectively running asynchronously on the remote node, with the only serialization point being the spawning done by the supervisor. - -For our routing layer, we are going to use tasks, but feel free to explore the other alternatives too. - -## async/await - -So far we have explored tasks that are started and run in isolation, with no regard for their return value. However, sometimes it is useful to run a task to compute a value and read its result later on. For this, tasks also provide the `async/await` pattern: - -```elixir -task = Task.async(fn -> compute_something_expensive end) -res = compute_something_else() -res + Task.await(task) -``` - -`async/await` provides a very simple mechanism to compute values concurrently. Not only that, `async/await` can also be used with the same [`Task.Supervisor`](https://hexdocs.pm/elixir/Task.Supervisor.html) we have used in previous chapters. We just need to call `Task.Supervisor.async/2` instead of `Task.Supervisor.start_child/2` and use `Task.await/2` to read the result later on. - -## Distributed tasks - -Distributed tasks are exactly the same as supervised tasks. The only difference is that we pass the node name when spawning the task on the supervisor. Open up `lib/kv/supervisor.ex` from the `:kv` application. Let's add a task supervisor as the last child of the tree: - -```elixir -supervisor(Task.Supervisor, [[name: KV.RouterTasks]]), -``` - -Now, let's start two named nodes again, but inside the `:kv` application: - -```bash -$ iex --sname foo -S mix -$ iex --sname bar -S mix -``` - -From inside `bar@computer-name`, we can now spawn a task directly on the other node via the supervisor: - -```iex -iex> task = Task.Supervisor.async {KV.RouterTasks, :"foo@computer-name"}, fn -> -...> {:ok, node()} -...> end -%Task{pid: #PID<12467.88.0>, ref: #Reference<0.0.0.400>} -iex> Task.await(task) -{:ok, :"foo@computer-name"} -``` - -Our first distributed task retrieves the name of the node the task is running on. Notice we have given an anonymous function to `Task.Supervisor.async/2` but, in distributed cases, it is preferable to give the module, function and arguments explicitly: - -```iex -iex> task = Task.Supervisor.async {KV.RouterTasks, :"foo@computer-name"}, Kernel, :node, [] -%Task{pid: #PID<12467.88.0>, ref: #Reference<0.0.0.400>} -iex> Task.await(task) -:"foo@computer-name" -``` - -The difference is that anonymous functions requires the target node to have exactly the same code version as the caller. Using module, function and arguments is more robust because you only need to find a function with matching arity in the given module. - -With this knowledge in hand, let's finally write the routing code. - -## Routing layer - -Create a file at `lib/kv/router.ex` with the following contents: - -```elixir -defmodule KV.Router do - @doc """ - Dispatch the given `mod`, `fun`, `args` request - to the appropriate node based on the `bucket`. - """ - def route(bucket, mod, fun, args) do - # Get the first byte of the binary - first = :binary.first(bucket) - - # Try to find an entry in the table or raise - entry = - Enum.find(table, fn {enum, _node} -> - first in enum - end) || no_entry_error(bucket) - - # If the entry node is the current node - if elem(entry, 1) == node() do - apply(mod, fun, args) - else - {KV.RouterTasks, elem(entry, 1)} - |> Task.Supervisor.async(KV.Router, :route, [bucket, mod, fun, args]) - |> Task.await() - end - end - - defp no_entry_error(bucket) do - raise "could not find entry for #{inspect bucket} in table #{inspect table}" - end - - @doc """ - The routing table. - """ - def table do - # Replace computer-name with your local machine name. - [{?a..?m, :"foo@computer-name"}, - {?n..?z, :"bar@computer-name"}] - end -end -``` - -Let's write a test to verify our router works. Create a file named `test/kv/router_test.exs` containing: - -```elixir -defmodule KV.RouterTest do - use ExUnit.Case, async: true - - test "route requests across nodes" do - assert KV.Router.route("hello", Kernel, :node, []) == - :"foo@computer-name" - assert KV.Router.route("world", Kernel, :node, []) == - :"bar@computer-name" - end - - test "raises on unknown entries" do - assert_raise RuntimeError, ~r/could not find entry/, fn -> - KV.Router.route(<<0>>, Kernel, :node, []) - end - end -end -``` - -The first test invokes `Kernel.node/0`, which returns the name of the current node, based on the bucket names "hello" and "world". According to our routing table so far, we should get `foo@computer-name` and `bar@computer-name` as responses, respectively. - -The second test checks that the code raises for unknown entries. - -In order to run the first test, we need to have two nodes running. Move into `apps/kv` and let's restart the node named `bar` which is going to be used by tests. - -```bash -$ iex --sname bar -S mix -``` - -And now run tests with: - -```bash -$ elixir --sname foo -S mix test -``` - -Our test should successfully pass. Excellent! - -## Test filters and tags - -Although our tests pass, our testing structure is getting more complex. In particular, running tests with only `mix test` causes failures in our suite, since our test requires a connection to another node. - -Luckily, ExUnit ships with a facility to tag tests, allowing us to run specific callbacks or even filter tests altogether based on those tags. We have already used the `:capture_log` tag in the previous chapter, which has its semantics specified by ExUnit itself. - -This time let's add a `:distributed` tag to `test/kv/router_test.exs`: - -```elixir -@tag :distributed -test "route requests across nodes" do -``` - -Writing `@tag :distributed` is equivalent to writing `@tag distributed: true`. - -With the test properly tagged, we can now check if the node is alive on the network and, if not, we can exclude all distributed tests. Open up `test/test_helper.exs` inside the `:kv` application and add the following: - -```elixir -exclude = - if Node.alive?, do: [], else: [distributed: true] - -ExUnit.start(exclude: exclude) -``` - -Now run tests with `mix test`: - -```bash -$ mix test -Excluding tags: [distributed: true] - -....... - -Finished in 0.1 seconds (0.1s on load, 0.01s on tests) -7 tests, 0 failures, 1 skipped -``` - -This time all tests passed and ExUnit warned us that distributed tests were being excluded. If you run tests with `$ elixir --sname foo -S mix test`, one extra test should run and successfully pass as long as the `bar@computer-name` node is available. - -The `mix test` command also allows us to dynamically include and exclude tags. For example, we can run `$ mix test --include distributed` to run distributed tests regardless of the value set in `test/test_helper.exs`. We could also pass `--exclude` to exclude a particular tag from the command line. Finally, `--only` can be used to run only tests with a particular tag: - -```bash -$ elixir --sname foo -S mix test --only distributed -``` - -You can read more about filters, tags and the default tags in [`ExUnit.Case` module documentation](https://hexdocs.pm/ex_unit/ExUnit.Case.html). - -## Application environment and configuration - -So far we have hardcoded the routing table into the `KV.Router` module. However, we would like to make the table dynamic. This allows us not only to configure development/test/production, but also to allow different nodes to run with different entries in the routing table. There is a feature of OTP that does exactly that: the application environment. - -Each application has an environment that stores the application's specific configuration by key. For example, we could store the routing table in the `:kv` application environment, giving it a default value and allowing other applications to change the table as needed. - -Open up `apps/kv/mix.exs` and change the `application/0` function to return the following: - -```elixir -def application do - [applications: [], - env: [routing_table: []], - mod: {KV, []}] -end -``` - -We have added a new `:env` key to the application. It returns the application default environment, which has an entry of key `:routing_table` and value of an empty list. It makes sense for the application environment to ship with an empty table, as the specific routing table depends on the testing/deployment structure. - -In order to use the application environment in our code, we need to replace `KV.Router.table/0` with the definition below: - -```elixir -@doc """ -The routing table. -""" -def table do - Application.fetch_env!(:kv, :routing_table) -end -``` - -We use `Application.fetch_env!/2` to read the entry for `:routing_table` in `:kv`'s environment. You can find more information and other functions to manipulate the app environment in the [Application module](https://hexdocs.pm/elixir/Application.html). - -Since our routing table is now empty, our distributed test should fail. Restart the apps and re-run tests to see the failure: - -```bash -$ iex --sname bar -S mix -$ elixir --sname foo -S mix test --only distributed -``` - -The interesting thing about the application environment is that it can be configured not only for the current application, but for all applications. Such configuration is done by the `config/config.exs` file. For example, we can configure IEx default prompt to another value. Just open `apps/kv/config/config.exs` and add the following to the end: - -```elixir -config :iex, default_prompt: ">>>" -``` - -Start IEx with `iex -S mix` and you can see that the IEx prompt has changed. - -This means we can also configure our `:routing_table` directly in the `apps/kv/config/config.exs` file: - -```elixir -# Replace computer-name with your local machine nodes. -config :kv, :routing_table, - [{?a..?m, :"foo@computer-name"}, - {?n..?z, :"bar@computer-name"}] -``` - -Restart the nodes and run distributed tests again. Now they should all pass. - -Since Elixir v1.2, all umbrella applications share their configurations, thanks to this line in `config/config.exs` in the umbrella root that loads the configuration of all children: - -```elixir -import_config "../apps/*/config/config.exs" -``` - -The `mix run` command also accepts a `--config` flag, which allows configuration files to be given on demand. This could be used to start different nodes, each with its own specific configuration (for example, different routing tables). - -Overall, the built-in ability to configure applications and the fact that we have built our software as an umbrella application gives us plenty of options when deploying the software. We can: - -* deploy the umbrella application to a node that will work as both TCP server and key-value storage - -* deploy the `:kv_server` application to work only as a TCP server as long as the routing table points only to other nodes - -* deploy only the `:kv` application when we want a node to work only as storage (no TCP access) - -As we add more applications in the future, we can continue controlling our deploy with the same level of granularity, cherry-picking which applications with which configuration are going to production. - -You can also consider building multiple releases with a tool like [exrm](https://github.com/bitwalker/exrm), which will package the chosen applications and configuration, including the current Erlang and Elixir installations, so we can deploy the application even if the runtime is not pre-installed on the target system. - -Finally, we have learned some new things in this chapter, and they could be applied to the `:kv_server` application as well. We are going to leave the next steps as an exercise: - -* change the `:kv_server` application to read the port from its application environment instead of using the hardcoded value of 4040 - -* change and configure the `:kv_server` application to use the routing functionality instead of dispatching directly to the local `KV.Registry`. For `:kv_server` tests, you can make the routing table point to the current node itself - -## Summing up - -In this chapter we have built a simple router as a way to explore the distributed features of Elixir and the Erlang VM, and learned how to configure its routing table. This is the last chapter in our Mix and OTP guide. - -Throughout the guide, we have built a very simple distributed key-value store as an opportunity to explore many constructs like generic servers, supervisors, tasks, agents, applications and more. Not only that, we have written tests for the whole application, got familiar with ExUnit, and learned how to use the Mix build tool to accomplish a wide range of tasks. - -If you are looking for a distributed key-value store to use in production, you should definitely look into [Riak](http://basho.com/riak/), which also runs in the Erlang VM. In Riak, the buckets are replicated, to avoid data loss, and instead of a router, they use [consistent hashing](https://en.wikipedia.org/wiki/Consistent_hashing) to map a bucket to a node. A consistent hashing algorithm helps reduce the amount of data that needs to be migrated when new nodes to store buckets are added to your infrastructure. - -There are many more lessons to learn and we hope you had fun so far! diff --git a/getting-started/mix-otp/distributed-tasks.markdown b/getting-started/mix-otp/distributed-tasks.markdown new file mode 100644 index 000000000..a75f95d9a --- /dev/null +++ b/getting-started/mix-otp/distributed-tasks.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: distributed-tasks +--- diff --git a/getting-started/mix-otp/docs-tests-and-with.markdown b/getting-started/mix-otp/docs-tests-and-with.markdown index 727a66176..dbeb73254 100644 --- a/getting-started/mix-otp/docs-tests-and-with.markdown +++ b/getting-started/mix-otp/docs-tests-and-with.markdown @@ -1,449 +1,5 @@ --- -layout: getting-started -title: Docs, tests and with -redirect_from: /getting-started/mix_otp/docs-tests-and-pipelines.html +layout: redirect +sitemap: false +redirect_to: docs-tests-and-with --- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -In this chapter, we will implement the code that parses the commands we described in the first chapter: - -``` -CREATE shopping -OK - -PUT shopping milk 1 -OK - -PUT shopping eggs 3 -OK - -GET shopping milk -1 -OK - -DELETE shopping eggs -OK -``` - -After the parsing is done, we will update our server to dispatch the parsed commands to the `:kv` application we built previously. - -## Doctests - -On the language homepage, we mention that Elixir makes documentation a first-class citizen in the language. We have explored this concept many times throughout this guide, be it via `mix help` or by typing `h Enum` or another module in an IEx console. - -In this section, we will implement the parse functionality using doctests, which allows us to write tests directly from our documentation. This helps us provide documentation with accurate code samples. - -Let's create our command parser at `lib/kv_server/command.ex` and start with the doctest: - -```elixir -defmodule KVServer.Command do - @doc ~S""" - Parses the given `line` into a command. - - ## Examples - - iex> KVServer.Command.parse "CREATE shopping\r\n" - {:ok, {:create, "shopping"}} - - """ - def parse(line) do - :not_implemented - end -end -``` - -Doctests are specified in by an indentation of four spaces followed by the `iex>` prompt in a documentation string. If a command spans multiple lines, you can use `...>`, as in IEx. The expected result should start at the next line after `iex>` or `...>` line(s) and is terminated either by a newline or a new `iex>` prefix. - -Also note that we started the documentation string using `@doc ~S"""`. The `~S` prevents the `\r\n` characters from being converted to a carriage return and line feed until they are evaluated in the test. - -To run our doctests, we'll create a file at `test/kv_server/command_test.exs` and call `doctest KVServer.Command` in the test case: - -```elixir -defmodule KVServer.CommandTest do - use ExUnit.Case, async: true - doctest KVServer.Command -end -``` - -Run the test suite and the doctest should fail: - -``` - 1) test doc at KVServer.Command.parse/1 (1) (KVServer.CommandTest) - test/kv_server/command_test.exs:3 - Doctest failed - code: KVServer.Command.parse "CREATE shopping\r\n" === {:ok, {:create, "shopping"}} - lhs: :not_implemented - stacktrace: - lib/kv_server/command.ex:11: KVServer.Command (module) -``` - -Excellent! - -Now let's make the doctest pass. Let's implement the `parse/1` function: - -```elixir -def parse(line) do - case String.split(line) do - ["CREATE", bucket] -> {:ok, {:create, bucket}} - end -end -``` - -Our implementation splits the line on whitespace and then matches the command against a list. Using `String.split/1` means our commands will be whitespace-insensitive. Leading and trailing whitespace won't matter, nor will consecutive spaces between words. Let's add some new doctests to test this behaviour along with the other commands: - -```elixir -@doc ~S""" -Parses the given `line` into a command. - -## Examples - - iex> KVServer.Command.parse "CREATE shopping\r\n" - {:ok, {:create, "shopping"}} - - iex> KVServer.Command.parse "CREATE shopping \r\n" - {:ok, {:create, "shopping"}} - - iex> KVServer.Command.parse "PUT shopping milk 1\r\n" - {:ok, {:put, "shopping", "milk", "1"}} - - iex> KVServer.Command.parse "GET shopping milk\r\n" - {:ok, {:get, "shopping", "milk"}} - - iex> KVServer.Command.parse "DELETE shopping eggs\r\n" - {:ok, {:delete, "shopping", "eggs"}} - -Unknown commands or commands with the wrong number of -arguments return an error: - - iex> KVServer.Command.parse "UNKNOWN shopping eggs\r\n" - {:error, :unknown_command} - - iex> KVServer.Command.parse "GET shopping\r\n" - {:error, :unknown_command} - -""" -``` - -With doctests at hand, it is your turn to make tests pass! Once you're ready, you can compare your work with our solution below: - -```elixir -def parse(line) do - case String.split(line) do - ["CREATE", bucket] -> {:ok, {:create, bucket}} - ["GET", bucket, key] -> {:ok, {:get, bucket, key}} - ["PUT", bucket, key, value] -> {:ok, {:put, bucket, key, value}} - ["DELETE", bucket, key] -> {:ok, {:delete, bucket, key}} - _ -> {:error, :unknown_command} - end -end -``` - -Notice how we were able to elegantly parse the commands without adding a bunch of `if/else` clauses that check the command name and number of arguments! - -Finally, you may have observed that each doctest was considered to be a different test in our test case, as our test suite now reports a total of 7 tests. That is because ExUnit considers the following to define two different tests: - -```iex -iex> KVServer.Command.parse "UNKNOWN shopping eggs\r\n" -{:error, :unknown_command} - -iex> KVServer.Command.parse "GET shopping\r\n" -{:error, :unknown_command} -``` - -Without new lines, as seen below, ExUnit compiles it into a single test: - -```iex -iex> KVServer.Command.parse "UNKNOWN shopping eggs\r\n" -{:error, :unknown_command} -iex> KVServer.Command.parse "GET shopping\r\n" -{:error, :unknown_command} -``` - -You can read more about doctests in [the `ExUnit.DocTest` docs](https://hexdocs.pm/ex_unit/ExUnit.DocTest.html). - -## with - -As we are now able to parse commands, we can finally start implementing the logic that runs the commands. Let's add a stub definition for this function for now: - -```elixir -defmodule KVServer.Command do - @doc """ - Runs the given command. - """ - def run(command) do - {:ok, "OK\r\n"} - end -end -``` - -Before we implement this function, let's change our server to start using our new `parse/1` and `run/1` functions. Remember, our `read_line/1` function was also crashing when the client closed the socket, so let's take the opportunity to fix it, too. Open up `lib/kv_server.ex` and replace the existing server definition: - -```elixir -defp serve(socket) do - socket - |> read_line() - |> write_line(socket) - - serve(socket) -end - -defp read_line(socket) do - {:ok, data} = :gen_tcp.recv(socket, 0) - data -end - -defp write_line(line, socket) do - :gen_tcp.send(socket, line) -end -``` - -by the following: - -```elixir -defp serve(socket) do - msg = - case read_line(socket) do - {:ok, data} -> - case KVServer.Command.parse(data) do - {:ok, command} -> - KVServer.Command.run(command) - {:error, _} = err -> - err - end - {:error, _} = err -> - err - end - - write_line(socket, msg) - serve(socket) -end - -defp read_line(socket) do - :gen_tcp.recv(socket, 0) -end - -defp write_line(socket, {:ok, text}) do - :gen_tcp.send(socket, text) -end - -defp write_line(socket, {:error, :unknown_command}) do - # Known error. Write to the client. - :gen_tcp.send(socket, "UNKNOWN COMMAND\r\n") -end - -defp write_line(_socket, {:error, :closed}) do - # The connection was closed, exit politely. - exit(:shutdown) -end - -defp write_line(socket, {:error, error}) do - # Unknown error. Write to the client and exit. - :gen_tcp.send(socket, "ERROR\r\n") - exit(error) -end -``` - -If we start our server, we can now send commands to it. For now we will get two different responses: "OK" when the command is known and "UNKNOWN COMMAND" otherwise: - -```bash -$ telnet 127.0.0.1 4040 -Trying 127.0.0.1... -Connected to localhost. -Escape character is '^]'. -CREATE shopping -OK -HELLO -UNKNOWN COMMAND -``` - -This means our implementation is going in the correct direction, but it doesn't look very elegant, does it? - -The previous implementation used pipelines which made the logic straight-forward to follow. However, now that we need to handle different error codes along the way, our server logic is nested inside many `case` calls. - -Thankfully, Elixir v1.2 introduced the `with` construct, which allows you to simplify code like the above, replacing nested `case` calls with a chain of matching clauses. Let's rewrite the `serve/1` function to use `with`: - -```elixir -defp serve(socket) do - msg = - with {:ok, data} <- read_line(socket), - {:ok, command} <- KVServer.Command.parse(data), - do: KVServer.Command.run(command) - - write_line(socket, msg) - serve(socket) -end -``` - -Much better! Syntax-wise, `with` is quite similar to `for` comprehensions. `with` will retrieve the value returned by the right-side of `<-` and match it against the pattern on the left side. If the value matches the pattern, `with` moves on to the next expression. In case there is no match, the non-matching value is returned. - -In other words, we converted each expression given to `case/2` as a step in `with`. As soon as any of the steps return something that does not match `{:ok, x}`, `with` aborts, and returns the non-matching value. - -You can read more about [`with` in our documentation](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#with/1). - -## Running commands - -The last step is to implement `KVServer.Command.run/1`, to run the parsed commands against the `:kv` application. Its implementation is shown below: - -```elixir -@doc """ -Runs the given command. -""" -def run(command) - -def run({:create, bucket}) do - KV.Registry.create(KV.Registry, bucket) - {:ok, "OK\r\n"} -end - -def run({:get, bucket, key}) do - lookup bucket, fn pid -> - value = KV.Bucket.get(pid, key) - {:ok, "#{value}\r\nOK\r\n"} - end -end - -def run({:put, bucket, key, value}) do - lookup bucket, fn pid -> - KV.Bucket.put(pid, key, value) - {:ok, "OK\r\n"} - end -end - -def run({:delete, bucket, key}) do - lookup bucket, fn pid -> - KV.Bucket.delete(pid, key) - {:ok, "OK\r\n"} - end -end - -defp lookup(bucket, callback) do - case KV.Registry.lookup(KV.Registry, bucket) do - {:ok, pid} -> callback.(pid) - :error -> {:error, :not_found} - end -end -``` - -Every function clause dispatches the appropriate command to the `KV.Registry` server that we registered during the `:kv` application startup. Since our `:kv_server` depends on the `:kv` application, it is completely fine to depend on the services it provides. - -Note that we have also defined a private function named `lookup/2` to help with the common functionality of looking up a bucket and returning its `pid` if it exists, `{:error, :not_found}` otherwise. - -By the way, since we are now returning `{:error, :not_found}`, we should amend the `write_line/2` function in `KVServer` to print such error as well: - -```elixir -defp write_line(socket, {:error, :not_found}) do - :gen_tcp.send(socket, "NOT FOUND\r\n") -end -``` - -And our server functionality is almost complete! Only tests are missing. This time, we have left tests for last because there are some important considerations to be made. - -`KVServer.Command.run/1`'s implementation is sending commands directly to the server named `KV.Registry`, which is registered by the `:kv` application. This means this server is global and if we have two tests sending messages to it at the same time, our tests will conflict with each other (and likely fail). We need to decide between having unit tests that are isolated and can run asynchronously, or writing integration tests that work on top of the global state, but exercise our application's full stack as it is meant to be exercised in production. - -So far we have only written unit tests, typically testing a single module directly. However, in order to make `KVServer.Command.run/1` testable as a unit we would need to change its implementation to not send commands directly to the `KV.Registry` process but instead pass a server as argument. For example, we would need to change `run`'s signature to `def run(command, pid)` and then change all clauses accordingly: - -```elixir -def run({:create, bucket}, pid) do - KV.Registry.create(pid, bucket) - {:ok, "OK\r\n"} -end - -# ... other run clauses ... -``` - -Feel free to go ahead and do the changes above and write some unit tests. The idea is that your tests will start an instance of the `KV.Registry` and pass it as argument to `run/2` instead of relying on the global `KV.Registry`. This has the advantage of keeping our tests asynchronous as there is no shared state. - -Since this is the approach we have done so far in our tests, we will try something different. Let's write integration tests that rely on the global server names to exercise the whole stack from the TCP server to the bucket. Our integration tests will rely on global state and must be synchronous. With integration tests we get coverage on how the components in our application work together at the cost of test performance. They are typically used to test the main flows in your application. For example, we should avoid using integration tests to test an edge case in our command parsing implementation. - -Our integration test will use a TCP client that sends commands to our server and assert we are getting the desired responses. - -Let's implement the integration test in `test/kv_server_test.exs` as shown below: - -```elixir -defmodule KVServerTest do - use ExUnit.Case - - setup do - Application.stop(:kv) - :ok = Application.start(:kv) - end - - setup do - opts = [:binary, packet: :line, active: false] - {:ok, socket} = :gen_tcp.connect('localhost', 4040, opts) - {:ok, socket: socket} - end - - test "server interaction", %{socket: socket} do - assert send_and_recv(socket, "UNKNOWN shopping\r\n") == - "UNKNOWN COMMAND\r\n" - - assert send_and_recv(socket, "GET shopping eggs\r\n") == - "NOT FOUND\r\n" - - assert send_and_recv(socket, "CREATE shopping\r\n") == - "OK\r\n" - - assert send_and_recv(socket, "PUT shopping eggs 3\r\n") == - "OK\r\n" - - # GET returns two lines - assert send_and_recv(socket, "GET shopping eggs\r\n") == "3\r\n" - assert send_and_recv(socket, "") == "OK\r\n" - - assert send_and_recv(socket, "DELETE shopping eggs\r\n") == - "OK\r\n" - - # GET returns two lines - assert send_and_recv(socket, "GET shopping eggs\r\n") == "\r\n" - assert send_and_recv(socket, "") == "OK\r\n" - end - - defp send_and_recv(socket, command) do - :ok = :gen_tcp.send(socket, command) - {:ok, data} = :gen_tcp.recv(socket, 0, 1000) - data - end -end -``` - -Our integration test checks all server interaction, including unknown commands and not found errors. It is worth noting that, as with ETS tables and linked processes, there is no need to close the socket. Once the test process exits, the socket is automatically closed. - -This time, since our test relies on global data, we have not given `async: true` to `use ExUnit.Case`. Furthermore, in order to guarantee our test is always in a clean state, we stop and start the `:kv` application before each test. In fact, stopping the `:kv` application even prints a warning on the terminal: - -``` -18:12:10.698 [info] Application kv exited: :stopped -``` - -To avoid printing log messages during tests, ExUnit provides a neat feature called `:capture_log`. By setting `@tag :capture_log` before each test or `@moduletag :capture_log` for the whole test case, ExUnit will automatically capture anything that is logged while the test runs. In case our test fails, the captured logs will be printed alongside the ExUnit report. - -Before setup, add the following call: - -```elixir -@moduletag :capture_log -``` - -In case the test crashes, you will see a report as follows: - -``` - 1) test server interaction (KVServerTest) - test/kv_server_test.exs:17 - ** (RuntimeError) oops - stacktrace: - test/kv_server_test.exs:29 - - The following output was logged: - - 13:44:10.035 [info] Application kv exited: :stopped -``` - -With this simple integration test, we start to see why integration tests may be slow. Not only can this particular test not run asynchronously, it also requires the expensive setup of stopping and starting the `:kv` application. - -At the end of the day, it is up to you and your team to figure out the best testing strategy for your applications. You need to balance code quality, confidence, and test suite runtime. For example, we may start with testing the server only with integration tests, but if the server continues to grow in future releases, or it becomes a part of the application with frequent bugs, it is important to consider breaking it apart and writing more intensive unit tests that don't have the weight of an integration test. - -In the next chapter we will finally make our system distributed by adding a bucket routing mechanism. We'll also learn about application configuration. diff --git a/getting-started/mix-otp/dynamic-supervisor.markdown b/getting-started/mix-otp/dynamic-supervisor.markdown new file mode 100644 index 000000000..9dfcd0300 --- /dev/null +++ b/getting-started/mix-otp/dynamic-supervisor.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: dynamic-supervisor +--- diff --git a/getting-started/mix-otp/ets.markdown b/getting-started/mix-otp/ets.markdown index 6eae963e2..22e0a0978 100644 --- a/getting-started/mix-otp/ets.markdown +++ b/getting-started/mix-otp/ets.markdown @@ -1,266 +1,5 @@ --- -layout: getting-started -title: ETS +layout: redirect +sitemap: false +redirect_to: erlang-term-storage --- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -Every time we need to look up a bucket, we need to send a message to the registry. In case our registry is being accessed concurrently by multiple processes, the registry may become a bottleneck! - -In this chapter we will learn about ETS (Erlang Term Storage) and how to use it as a cache mechanism. - -> Warning! Don't use ETS as a cache prematurely! Log and analyze your application performance and identify which parts are bottlenecks, so you know *whether* you should cache, and *what* you should cache. This chapter is merely an example of how ETS can be used, once you've determined the need. - -## ETS as a cache - -ETS allows us to store any Elixir term in an in-memory table. Working with ETS tables is done via [Erlang's `:ets` module](http://www.erlang.org/doc/man/ets.html): - -```iex -iex> table = :ets.new(:buckets_registry, [:set, :protected]) -8207 -iex> :ets.insert(table, {"foo", self}) -true -iex> :ets.lookup(table, "foo") -[{"foo", #PID<0.41.0>}] -``` - -When creating an ETS table, two arguments are required: the table name and a set of options. From the available options, we passed the table type and its access rules. We have chosen the `:set` type, which means that keys cannot be duplicated. We've also set the table's access to `:protected`, meaning only the process that created the table can write to it, but all processes can read from it. Those are actually the default values, so we will skip them from now on. - -ETS tables can also be named, allowing us to access them by a given name: - -```iex -iex> :ets.new(:buckets_registry, [:named_table]) -:buckets_registry -iex> :ets.insert(:buckets_registry, {"foo", self}) -true -iex> :ets.lookup(:buckets_registry, "foo") -[{"foo", #PID<0.41.0>}] -``` - -Let's change the `KV.Registry` to use ETS tables. Since our registry requires a name as argument, we are going to name the ETS table with the same name as the registry. ETS names and process names are stored in different locations, so there is no chance of conflicts. - -Open up `lib/kv/registry.ex`, and let's change its implementation. We've added comments to the source code to highlight the changes we've made: - -```elixir -defmodule KV.Registry do - use GenServer - - ## Client API - - @doc """ - Starts the registry with the given `name`. - """ - def start_link(name) do - # 1. Pass the name to GenServer's init - GenServer.start_link(__MODULE__, name, name: name) - end - - @doc """ - Looks up the bucket pid for `name` stored in `server`. - - Returns `{:ok, pid}` if the bucket exists, `:error` otherwise. - """ - def lookup(server, name) when is_atom(server) do - # 2. Lookup is now done directly in ETS, without accessing the server - case :ets.lookup(server, name) do - [{^name, pid}] -> {:ok, pid} - [] -> :error - end - end - - @doc """ - Ensures there is a bucket associated to the given `name` in `server`. - """ - def create(server, name) do - GenServer.cast(server, {:create, name}) - end - - @doc """ - Stops the registry. - """ - def stop(server) do - GenServer.stop(server) - end - - ## Server callbacks - - def init(table) do - # 3. We have replaced the names map by the ETS table - names = :ets.new(table, [:named_table, read_concurrency: true]) - refs = %{} - {:ok, {names, refs}} - end - - # 4. The previous handle_call callback for lookup was removed - - def handle_cast({:create, name}, {names, refs}) do - # 5. Read and write to the ETS table instead of the map - case lookup(names, name) do - {:ok, _pid} -> - {:noreply, {names, refs}} - :error -> - {:ok, pid} = KV.Bucket.Supervisor.start_bucket - ref = Process.monitor(pid) - refs = Map.put(refs, ref, name) - :ets.insert(names, {name, pid}) - {:noreply, {names, refs}} - end - end - - def handle_info({:DOWN, ref, :process, _pid, _reason}, {names, refs}) do - # 6. Delete from the ETS table instead of the map - {name, refs} = Map.pop(refs, ref) - :ets.delete(names, name) - {:noreply, {names, refs}} - end - - def handle_info(_msg, state) do - {:noreply, state} - end -end -``` - -Notice that before our changes `KV.Registry.lookup/2` sent requests to the server, but now it reads directly from the ETS table, which is shared across all processes. That's the main idea behind the cache mechanism we are implementing. - -In order for the cache mechanism to work, the created ETS table needs to have access `:protected` (the default), so all clients can read from it, while only the `KV.Registry` process writes to it. We have also set `read_concurrency: true` when starting the table, optimizing the table for the common scenario of concurrent read operations. - -The changes we have performed above have broken our tests because they were using the pid of the registry process for all operations and now the registry lookup requires the ETS table name. However, since the ETS table has the same name as the registry process, it is an easy fix. Change the setup function in `test/kv/registry_test.exs` to the following: - -```elixir -setup context do - {:ok, _} = KV.Registry.start_link(context.test) - {:ok, registry: context.test} -end -``` - -Once we change `setup`, some tests will continue to fail. You may even notice tests pass and fail inconsistently between runs. For example, the "spawns buckets" test: - -```elixir -test "spawns buckets", %{registry: registry} do - assert KV.Registry.lookup(registry, "shopping") == :error - - KV.Registry.create(registry, "shopping") - assert {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - - KV.Bucket.put(bucket, "milk", 1) - assert KV.Bucket.get(bucket, "milk") == 1 -end -``` - -may be failing on this line: - -```elixir -{:ok, bucket} = KV.Registry.lookup(registry, "shopping") -``` - -How can this line fail if we just created the bucket in the previous line? - -The reason those failures are happening is because, for didactic purposes, we have made two mistakes: - - 1. We are prematurely optimizing (by adding this cache layer) - 2. We are using `cast/2` (while we should be using `call/2`) - -## Race conditions? - -Developing in Elixir does not make your code free of race conditions. However, Elixir's simple abstractions where nothing is shared by default make it easier to spot a race condition's root cause. - -What is happening in our tests is that there is a delay in between an operation and the time we can observe this change in the ETS table. Here is what we were expecting to happen: - -1. We invoke `KV.Registry.create(registry, "shopping")` -2. The registry creates the bucket and updates the cache table -3. We access the information from the table with `KV.Registry.lookup(registry, "shopping")` -4. The command above returns `{:ok, bucket}` - -However, since `KV.Registry.create/2` is a cast operation, the command will return before we actually write to the table! In other words, this is happening: - -1. We invoke `KV.Registry.create(registry, "shopping")` -2. We access the information from the table with `KV.Registry.lookup(registry, "shopping")` -3. The command above returns `:error` -4. The registry creates the bucket and updates the cache table - -To fix the failure we need to make `KV.Registry.create/2` synchronous by using `call/2` rather than `cast/2`. This will guarantee that the client will only continue after changes have been made to the table. Let's change the function and its callback as follows: - -```elixir -def create(server, name) do - GenServer.call(server, {:create, name}) -end - -def handle_call({:create, name}, _from, {names, refs}) do - case lookup(names, name) do - {:ok, pid} -> - {:reply, pid, {names, refs}} - :error -> - {:ok, pid} = KV.Bucket.Supervisor.start_bucket - ref = Process.monitor(pid) - refs = Map.put(refs, ref, name) - :ets.insert(names, {name, pid}) - {:reply, pid, {names, refs}} - end -end -``` - -We changed the callback from `handle_cast/2` to `handle_call/3` and changed it to reply with the pid of the created bucket. Generally speaking, Elixir developers prefer to use `call/2` instead of `cast/2` as it also provides back-pressure (you block until you get a reply). Using `cast/2` when not necessary can also be considered a premature optimization. - -Let's run the tests once again. This time though, we will pass the `--trace` option: - -```bash -$ mix test --trace -``` - -The `--trace` option is useful when your tests are deadlocking or there are race conditions, as it runs all tests synchronously (`async: true` has no effect) and shows detailed information about each test. This time we should be down to one or two intermittent failures: - -``` - 1) test removes buckets on exit (KV.RegistryTest) - test/kv/registry_test.exs:19 - Assertion with == failed - code: KV.Registry.lookup(registry, "shopping") == :error - lhs: {:ok, #PID<0.109.0>} - rhs: :error - stacktrace: - test/kv/registry_test.exs:23 -``` - -According to the failure message, we are expecting that the bucket no longer exists on the table, but it still does! This problem is the opposite of the one we have just solved: while previously there was a delay between the command to create a bucket and updating the table, now there is a delay between the bucket process dying and its entry being removed from the table. - -Unfortunately this time we cannot simply change `handle_info/2`, the operation responsible for cleaning the ETS table, to a synchronous operation. Instead we need to find a way to guarantee the registry has processed the `:DOWN` notification sent when the bucket crashed. - -An easy way to do so is by sending a synchronous request to the registry: because messages are processed in order, if the registry replies to a request sent after the `Agent.stop` call, it means that the `:DOWN` message has been processed. Let's do so by creating a "bogus" bucket, which is a synchronous request, after `Agent.stop` in both tests: - - -```elixir -test "removes buckets on exit", %{registry: registry} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - Agent.stop(bucket) - - # Do a call to ensure the registry processed the DOWN message - _ = KV.Registry.create(registry, "bogus") - assert KV.Registry.lookup(registry, "shopping") == :error -end - -test "removes bucket on crash", %{registry: registry} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - - # Kill the bucket and wait for the notification - Process.exit(bucket, :shutdown) - - # Wait until the bucket is dead - ref = Process.monitor(bucket) - assert_receive {:DOWN, ^ref, _, _, _} - - # Do a call to ensure the registry processed the DOWN message - _ = KV.Registry.create(registry, "bogus") - assert KV.Registry.lookup(registry, "shopping") == :error -end -``` - -Our tests should now (always) pass! - -This concludes our optimization chapter. We have used ETS as a cache mechanism where reads can happen from any processes but writes are still serialized through a single process. More importantly, we have also learned that once data can be read asynchronously, we need to be aware of the race conditions it might introduce. - -Next let's discuss external and internal dependencies and how Mix helps us manage large codebases. diff --git a/getting-started/mix-otp/genserver.markdown b/getting-started/mix-otp/genserver.markdown index 8aea292ed..0c548b6bb 100644 --- a/getting-started/mix-otp/genserver.markdown +++ b/getting-started/mix-otp/genserver.markdown @@ -1,267 +1,5 @@ --- -layout: getting-started -title: GenServer +layout: redirect +sitemap: false +redirect_to: genservers --- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -In the [previous chapter](/getting-started/mix-otp/agent.html) we used agents to represent our buckets. In the first chapter, we specified we would like to name each bucket so we can do the following: - -```elixir -CREATE shopping -OK - -PUT shopping milk 1 -OK - -GET shopping milk -1 -OK -``` - -Since agents are processes, each bucket has a process identifier (pid) but it doesn't have a name. We have learned about the name registry [in the Process chapter](/getting-started/processes.html) and you could be inclined to solve this problem by using such registry. For example, we could create a bucket as: - -```iex -iex> Agent.start_link(fn -> %{} end, name: :shopping) -{:ok, #PID<0.43.0>} -iex> KV.Bucket.put(:shopping, "milk", 1) -:ok -iex> KV.Bucket.get(:shopping, "milk") -1 -``` - -However, this is a terrible idea! Process names in Elixir must be atoms, which means we would need to convert the bucket name (often received from an external client) to atoms, and **we should never convert user input to atoms**. This is because atoms are not garbage collected. Once an atom is created, it is never reclaimed. Generating atoms from user input would mean the user can inject enough different names to exhaust our system memory! - -In practice it is more likely you will reach the Erlang VM limit for the maximum number of atoms before you run out of memory, which will bring your system down regardless. - -Instead of abusing the name registry facility, we will create our own *registry process* that holds a map that associates the bucket name to the bucket process. - -The registry needs to guarantee that the dictionary is always up to date. For example, if one of the bucket processes crashes due to a bug, the registry must clean up the dictionary in order to avoid serving stale entries. In Elixir, we describe this by saying that the registry needs to *monitor* each bucket. - -We will use a [GenServer](https://hexdocs.pm/elixir/GenServer.html) to create a registry process that can monitor the bucket processes. GenServers are the go-to abstraction for building generic servers in both Elixir and OTP. - -## Our first GenServer - -A GenServer is implemented in two parts: the client API and the server callbacks, either in a single module or in two different modules implementing client API in one and server callbacks in the other. The client and server run in separate processes, with the client passing messages back and forth to the server as its functions are called. Here we use a single module for both the server callbacks and client API. Create a new file at `lib/kv/registry.ex` with the following contents: - -```elixir -defmodule KV.Registry do - use GenServer - - ## Client API - - @doc """ - Starts the registry. - """ - def start_link do - GenServer.start_link(__MODULE__, :ok, []) - end - - @doc """ - Looks up the bucket pid for `name` stored in `server`. - - Returns `{:ok, pid}` if the bucket exists, `:error` otherwise. - """ - def lookup(server, name) do - GenServer.call(server, {:lookup, name}) - end - - @doc """ - Ensures there is a bucket associated to the given `name` in `server`. - """ - def create(server, name) do - GenServer.cast(server, {:create, name}) - end - - ## Server Callbacks - - def init(:ok) do - {:ok, %{}} - end - - def handle_call({:lookup, name}, _from, names) do - {:reply, Map.fetch(names, name), names} - end - - def handle_cast({:create, name}, names) do - if Map.has_key?(names, name) do - {:noreply, names} - else - {:ok, bucket} = KV.Bucket.start_link - {:noreply, Map.put(names, name, bucket)} - end - end -end -``` - -The first function is `start_link/3`, which starts a new GenServer passing three arguments: - -1. The module where the server callbacks are implemented, in this case `__MODULE__`, meaning the current module - -2. The initialization arguments, in this case the atom `:ok` - -3. A list of options which can, for example, hold the name of the server. For now, we pass an empty list - -There are two types of requests you can send to a GenServer: calls and casts. Calls are synchronous and the server **must** send a response back to such requests. Casts are asynchronous and the server won't send a response back. - -The next two functions, `lookup/2` and `create/2` are responsible for sending these requests to the server. The requests are represented by the first argument to `handle_call/3` or `handle_cast/2`. In this case, we have used `{:lookup, name}` and `{:create, name}` respectively. Requests are often specified as tuples, like this, in order to provide more than one "argument" in that first argument slot. It's common to specify the action being requested as the first element of a tuple, and arguments for that action in the remaining elements. - -On the server side, we can implement a variety of callbacks to guarantee the server initialization, termination and handling of requests. Those callbacks are optional and for now we have only implemented the ones we care about. - -The first is the `init/1` callback, that receives the argument given to `GenServer.start_link/3` and returns `{:ok, state}`, where state is a new map. We can already notice how the `GenServer` API makes the client/server segregation more apparent. `start_link/3` happens in the client, while `init/1` is the respective callback that runs on the server. - -For `call/2` requests, we must implement a `handle_call/3` callback that receives the `request`, the process from which we received the request (`_from`), and the current server state (`names`). The `handle_call/3` callback returns a tuple in the format `{:reply, reply, new_state}`, where `reply` is what will be sent to the client and the `new_state` is the new server state. - -For `cast/2` requests, we must implement a `handle_cast/2` callback that receives the `request` and the current server state (`names`). The `handle_cast/2` callback returns a tuple in the format `{:noreply, new_state}`. - -There are other tuple formats both `handle_call/3` and `handle_cast/2` callbacks may return. There are also other callbacks like `terminate/2` and `code_change/3` that we could implement. You are welcome to explore the [full GenServer documentation](https://hexdocs.pm/elixir/GenServer.html) to learn more about those. - -For now, let's write some tests to guarantee our GenServer works as expected. - -## Testing a GenServer - -Testing a GenServer is not much different from testing an agent. We will spawn the server on a setup callback and use it throughout our tests. Create a file at `test/kv/registry_test.exs` with the following: - -```elixir -defmodule KV.RegistryTest do - use ExUnit.Case, async: true - - setup do - {:ok, registry} = KV.Registry.start_link - {:ok, registry: registry} - end - - test "spawns buckets", %{registry: registry} do - assert KV.Registry.lookup(registry, "shopping") == :error - - KV.Registry.create(registry, "shopping") - assert {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - - KV.Bucket.put(bucket, "milk", 1) - assert KV.Bucket.get(bucket, "milk") == 1 - end -end -``` - -Our test should pass right out of the box! - -We don't need to explicitly shut down the registry because it will receive a `:shutdown` signal when our test finishes. While this solution is ok for tests, if there is a need to stop a `GenServer` as part of the application logic, one can use the `GenServer.stop/1` function: - -```elixir -## Client API - -@doc """ -Stops the registry. -""" -def stop(server) do - GenServer.stop(server) -end -``` - -## The need for monitoring - -Our registry is almost complete. The only remaining issue is that the registry may become stale if a bucket stops or crashes. Let's add a test to `KV.RegistryTest` that exposes this bug: - -```elixir -test "removes buckets on exit", %{registry: registry} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - Agent.stop(bucket) - assert KV.Registry.lookup(registry, "shopping") == :error -end -``` - -The test above will fail on the last assertion as the bucket name remains in the registry even after we stop the bucket process. - -In order to fix this bug, we need the registry to monitor every bucket it spawns. Once we set up a monitor, the registry will receive a notification every time a bucket exits, allowing us to clean the dictionary up. - -Let's first play with monitors by starting a new console with `iex -S mix`: - -```iex -iex> {:ok, pid} = KV.Bucket.start_link -{:ok, #PID<0.66.0>} -iex> Process.monitor(pid) -#Reference<0.0.0.551> -iex> Agent.stop(pid) -:ok -iex> flush -{:DOWN, #Reference<0.0.0.551>, :process, #PID<0.66.0>, :normal} -``` - -Note `Process.monitor(pid)` returns a unique reference that allows us to match upcoming messages to that monitoring reference. After we stop the agent, we can `flush/0` all messages and notice a `:DOWN` message arrived, with the exact reference returned by monitor, notifying that the bucket process exited with reason `:normal`. - -Let's reimplement the server callbacks to fix the bug and make the test pass. First, we will modify the GenServer state to two dictionaries: one that contains `name -> pid` and another that holds `ref -> name`. Then we need to monitor the buckets on `handle_cast/2` as well as implement a `handle_info/2` callback to handle the monitoring messages. The full server callbacks implementation is shown below: - -```elixir -## Server callbacks - -def init(:ok) do - names = %{} - refs = %{} - {:ok, {names, refs}} -end - -def handle_call({:lookup, name}, _from, {names, _} = state) do - {:reply, Map.fetch(names, name), state} -end - -def handle_cast({:create, name}, {names, refs}) do - if Map.has_key?(names, name) do - {:noreply, {names, refs}} - else - {:ok, pid} = KV.Bucket.start_link - ref = Process.monitor(pid) - refs = Map.put(refs, ref, name) - names = Map.put(names, name, pid) - {:noreply, {names, refs}} - end -end - -def handle_info({:DOWN, ref, :process, _pid, _reason}, {names, refs}) do - {name, refs} = Map.pop(refs, ref) - names = Map.delete(names, name) - {:noreply, {names, refs}} -end - -def handle_info(_msg, state) do - {:noreply, state} -end -``` - -Observe that we were able to considerably change the server implementation without changing any of the client API. That's one of the benefits of explicitly segregating the server and the client. - -Finally, different from the other callbacks, we have defined a "catch-all" clause for `handle_info/2` that discards any unknown message. To understand why, let's move on to the next section. - -## `call`, `cast` or `info`? - -So far we have used three callbacks: `handle_call/3`, `handle_cast/2` and `handle_info/2`. Here is what we should consider when deciding when to use each: - -1. `handle_call/3` must be used for synchronous requests. This should be the default choice as waiting for the server reply is a useful backpressure mechanism. - -2. `handle_cast/2` must be used for asynchronous requests, when you don't care about a reply. A cast does not even guarantee the server has received the message and, for this reason, must be used sparingly. For example, the `create/2` function we have defined in this chapter should have used `call/2`. We have used `cast/2` for didactic purposes. - -3. `handle_info/2` must be used for all other messages a server may receive that are not sent via `GenServer.call/2` or `GenServer.cast/2`, including regular messages sent with `send/2`. The monitoring `:DOWN` messages are such an example of this. - -Since any message, including the ones sent via `send/2`, go to `handle_info/2`, there is a chance unexpected messages will arrive to the server. Therefore, if we don't define the catch-all clause, those messages could lead our registry to crash, because no clause would match. We don't need to worry about such cases for `handle_call/3` and `handle_cast/2` though. Calls and casts are only done via the `GenServer` API, so an unknown message is quite likely to be due to a developer mistake. - -To help developers remember the differences between call, cast and info, the supported return values and more, [Benjamin Tan Wei Hao](http://benjamintan.io) has created an excellent [GenServer cheat sheet](https://raw.githubusercontent.com/benjamintanweihao/elixir-cheatsheets/master/GenServer_CheatSheet.pdf). - -## Monitors or links? - -We have previously learned about links in the [Process chapter](/getting-started/processes.html). Now, with the registry complete, you may be wondering: when should we use monitors and when should we use links? - -Links are bi-directional. If you link two processes and one of them crashes, the other side will crash too (unless it is trapping exits). A monitor is uni-directional: only the monitoring process will receive notifications about the monitored one. In other words: use links when you want linked crashes, and monitors when you just want to be informed of crashes, exits, and so on. - -Returning to our `handle_cast/2` implementation, you can see the registry is both linking and monitoring the buckets: - -```elixir -{:ok, pid} = KV.Bucket.start_link -ref = Process.monitor(pid) -``` - -This is a bad idea, as we don't want the registry to crash when a bucket crashes! We typically avoid creating new processes directly, instead we delegate this responsibility to supervisors. As we'll see in the next chapter, supervisors rely on links and that explains why link-based APIs (`spawn_link`, `start_link`, etc) are so prevalent in Elixir and OTP. - diff --git a/getting-started/mix-otp/index.html b/getting-started/mix-otp/index.html index 85516e00d..6406e2fed 100644 --- a/getting-started/mix-otp/index.html +++ b/getting-started/mix-otp/index.html @@ -1,5 +1,5 @@ - - - - - +--- +layout: redirect +sitemap: false +redirect_to: introduction-to-mix +--- diff --git a/getting-started/mix-otp/introduction-to-mix.markdown b/getting-started/mix-otp/introduction-to-mix.markdown index a5f1ca0e2..6406e2fed 100644 --- a/getting-started/mix-otp/introduction-to-mix.markdown +++ b/getting-started/mix-otp/introduction-to-mix.markdown @@ -1,271 +1,5 @@ --- -layout: getting-started -title: Introduction to Mix +layout: redirect +sitemap: false +redirect_to: introduction-to-mix --- - -# {{ page.title }} - -{% include toc.html %} - -In this guide, we will learn how to build a complete Elixir application, with its own supervision tree, configuration, tests and more. - -The application works as a distributed key-value store. We are going to organize key-value pairs into buckets and distribute those buckets across multiple nodes. We will also build a simple client that allows us to connect to any of those nodes and send requests such as: - -``` -CREATE shopping -OK - -PUT shopping milk 1 -OK - -PUT shopping eggs 3 -OK - -GET shopping milk -1 -OK - -DELETE shopping eggs -OK -``` - -In order to build our key-value application, we are going to use three main tools: - -* ***OTP*** _(Open Telecom Platform)_ is a set of libraries that ships with Erlang. Erlang developers use OTP to build robust, fault-tolerant applications. In this chapter we will explore how many aspects from OTP integrate with Elixir, including supervision trees, event managers and more; - -* ***Mix*** is a build tool that ships with Elixir that provides tasks for creating, compiling, testing your application, managing its dependencies and much more; - -* ***ExUnit*** is a test-unit based framework that ships with Elixir; - -In this chapter, we will create our first project using Mix and explore different features in OTP, Mix and ExUnit as we go. - -> Note: this guide requires Elixir v1.2.0 or later. You can check your Elixir version with `elixir -v` and install a more recent version if required by following the steps described in [the first chapter of the Getting Started guide](/getting-started/introduction.html). -> -> If you have any questions or improvements to the guide, please reach discussion channels such as the [Elixir Forum](https://elixirforum.com) or the [issues tracker](https://github.com/elixir-lang/elixir-lang.github.com/issues). Your input is really important to help us guarantee the guides are accessible and up to date! -> -> The final code for this guide is in [this repository](https://github.com/josevalim/kv_umbrella) and can be used as a reference. - -## Our first project - -When you install Elixir, besides getting the `elixir`, `elixirc` and `iex` executables, you also get an executable Elixir script named `mix`. - -Let's create our first project by invoking `mix new` from the command line. We'll pass the project name as argument (`kv`, in this case), and tell Mix that our main module should be the all-uppercase `KV`, instead of the default, which would have been `Kv`: - -```bash -$ mix new kv --module KV -``` - -Mix will create a directory named `kv` with a few files in it: - - * creating README.md - * creating .gitignore - * creating mix.exs - * creating config - * creating config/config.exs - * creating lib - * creating lib/kv.ex - * creating test - * creating test/test_helper.exs - * creating test/kv_test.exs - -Let's take a brief look at those generated files. - -> Note: Mix is an Elixir executable. This means that in order to run `mix`, you need to have Elixir's executable in your PATH. If not, you can run it by passing the script as argument to `elixir`: -> -> ```bash -> $ bin/elixir bin/mix new kv --module KV -> ``` -> -> Note that you can also execute any script in your PATH from Elixir via the -S option: -> -> ```bash -> $ bin/elixir -S mix new kv --module KV -> ``` -> -> When using -S, `elixir` finds the script wherever it is in your PATH and executes it. - -## Project compilation - -A file named `mix.exs` was generated inside our new project folder (`kv`) and its main responsibility is to configure our project. Let's take a look at it (comments removed): - -```elixir -defmodule KV.Mixfile do - use Mix.Project - - def project do - [app: :kv, - version: "0.1.0", - elixir: "~> 1.3", - build_embedded: Mix.env == :prod, - start_permanent: Mix.env == :prod, - deps: deps()] - end - - def application do - [applications: [:logger]] - end - - defp deps do - [] - end -end -``` - -Our `mix.exs` defines two public functions: `project`, which returns project configuration like the project name and version, and `application`, which is used to generate an application file. - -There is also a private function named `deps`, which is invoked from the `project` function, that defines our project dependencies. Defining `deps` as a separate function is not required, but it helps keep the project configuration tidy. - -Mix also generates a file at `lib/kv.ex` with a simple module definition: - -```elixir -defmodule KV do -end -``` - -This structure is enough to compile our project: - -```bash -$ cd kv -$ mix compile -``` - -Will output: - - Compiling 1 file (.ex) - Generated kv app - -The `lib/kv.ex` file was compiled, an application manifest named `kv.app` was generated and [all protocols were consolidated as described in the Getting Started guide](/getting-started/protocols.html#protocol-consolidation). All compilation artifacts are placed inside the `_build` directory using the options defined in the `mix.exs` file. - -Once the project is compiled, you can start an `iex` session inside the project by running: - -```bash -$ iex -S mix -``` - -## Running tests - -Mix also generated the appropriate structure for running our project tests. Mix projects usually follow the convention of having a `_test.exs` file in the `test` directory for each file in the `lib` directory. For this reason, we can already find a `test/kv_test.exs` corresponding to our `lib/kv.ex` file. It doesn't do much at this point: - -```elixir -defmodule KVTest do - use ExUnit.Case - doctest KV - - test "the truth" do - assert 1 + 1 == 2 - end -end -``` - -It is important to note a couple things: - -1. the test file is an Elixir script file (`.exs`). This is convenient because we don't need to compile test files before running them; - -2. we define a test module named `KVTest`, use [`ExUnit.Case`](https://hexdocs.pm/ex_unit/ExUnit.Case.html) to inject the testing API and define a simple test using the `test/2` macro; - -Mix also generated a file named `test/test_helper.exs` which is responsible for setting up the test framework: - -```elixir -ExUnit.start() -``` - -This file will be automatically required by Mix every time before we run our tests. We can run tests with `mix test`: - - Compiled lib/kv.ex - Generated kv app - [...] - . - - Finished in 0.04 seconds (0.04s on load, 0.00s on tests) - 1 test, 0 failures - - Randomized with seed 540224 - -Notice that by running `mix test`, Mix has compiled the source files and generated the application file once again. This happens because Mix supports multiple environments, which we will explore in the next section. - -Furthermore, you can see that ExUnit prints a dot for each successful test and automatically randomizes tests too. Let's make the test fail on purpose and see what happens. - -Change the assertion in `test/kv_test.exs` to the following: - -```elixir -assert 1 + 1 == 3 -``` - -Now run `mix test` again (notice this time there will be no compilation): - -``` - 1) test the truth (KVTest) - test/kv_test.exs:5 - Assertion with == failed - code: 1 + 1 == 3 - lhs: 2 - rhs: 3 - stacktrace: - test/kv_test.exs:6 - -Finished in 0.05 seconds (0.05s on load, 0.00s on tests) -1 test, 1 failure -``` - -For each failure, ExUnit prints a detailed report, containing the test name with the test case, the code that failed and the values for the left-hand side (lhs) and right-hand side (rhs) of the `==` operator. - -In the second line of the failure, right below the test name, there is the location where the test was defined. If you copy the test location in this full second line (including the file and line number) and append it to `mix test`, Mix will load and run just that particular test: - -```bash -$ mix test test/kv_test.exs:5 -``` - -This shortcut will be extremely useful as we build our project, allowing us to quickly iterate by running just a specific test. - -Finally, the stacktrace relates to the failure itself, giving information about the test and often the place the failure was generated from within the source files. - -## Environments - -Mix supports the concept of "environments". They allow a developer to customize compilation and other options for specific scenarios. By default, Mix understands three environments: - -* `:dev` - the one in which Mix tasks (like `compile`) run by default -* `:test` - used by `mix test` -* `:prod` - the one you will use to run your project in production - -The environment applies only to the current project. As we will see later on, any dependency you add to your project will by default run in the `:prod` environment. - -Customization per environment can be done by accessing [the `Mix.env` function](https://hexdocs.pm/mix/Mix.html#env/1) in your `mix.exs` file, which returns the current environment as an atom. That's what we have used in both `:build_embedded` and `:start_permanent` options: - -```elixir -def project do - [..., - build_embedded: Mix.env == :prod, - start_permanent: Mix.env == :prod, - ...] -end -``` - -When you compile your source code, Elixir compiles artifacts to the `_build` directory. However, in many occasions to avoid unnecessary copying, Elixir will create filesystem links from `_build` to actual source files. When true, `:build_embedded` disables this behaviour as it aims to provide everything you need to run your application inside `_build`. - -Similarly, when true, the `:start_permanent` option starts your application in permanent mode, which means the Erlang VM will crash if your application's supervision tree shuts down. Notice we don't want this behaviour in dev and test because it is useful to keep the VM instance running in those environments for troubleshooting purposes. - -Mix will default to the `:dev` environment, except for the `test` task that will default to the `:test` environment. The environment can be changed via the `MIX_ENV` environment variable: - -```bash -$ MIX_ENV=prod mix compile -``` - -Or on Windows: - -```batch -> set "MIX_ENV=prod" && mix compile -``` - -## Exploring - -There is much more to Mix, and we will continue to explore it as we build our project. A [general overview is available on the Mix documentation](https://hexdocs.pm/mix/). - -Keep in mind that you can always invoke the help task to list all available tasks: - -```bash -$ mix help -``` - -You can get further information about a particular task by invoking `mix help TASK`. - -Let's write some code! diff --git a/getting-started/mix-otp/supervisor-and-application.markdown b/getting-started/mix-otp/supervisor-and-application.markdown index 0710b2ed1..2d073b41b 100644 --- a/getting-started/mix-otp/supervisor-and-application.markdown +++ b/getting-started/mix-otp/supervisor-and-application.markdown @@ -1,417 +1,5 @@ --- -layout: getting-started -title: Supervisor and Application +layout: redirect +sitemap: false +redirect_to: supervisor-and-application --- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -So far our application has a registry that may monitor dozens, if not hundreds, of buckets. While we think our implementation so far is quite good, no software is bug free, and failures are definitely going to happen. - -When things fail, your first reaction may be: "let's rescue those errors". But in Elixir we avoid the defensive programming habit of rescuing exceptions, as commonly seen in other languages. Instead, we say "let it crash". If there is a bug that leads our registry to crash, we have nothing to worry about because we are going to set up a supervisor that will start a fresh copy of the registry. - -In this chapter, we are going to learn about supervisors and also about applications. We are going to create not one, but two supervisors, and use them to supervise our processes. - -## Our first supervisor - -Creating a supervisor is not much different from creating a GenServer. We are going to define a module named `KV.Supervisor`, which will use the [Supervisor](https://hexdocs.pm/elixir/Supervisor.html) behaviour, inside the `lib/kv/supervisor.ex` file: - -```elixir -defmodule KV.Supervisor do - use Supervisor - - def start_link do - Supervisor.start_link(__MODULE__, :ok) - end - - def init(:ok) do - children = [ - worker(KV.Registry, [KV.Registry]) - ] - - supervise(children, strategy: :one_for_one) - end -end -``` - -Our supervisor has a single child so far: the registry. A worker in the format of: - -```elixir -worker(KV.Registry, [KV.Registry]) -``` - -is going to start a process using the following call: - -```elixir -KV.Registry.start_link(KV.Registry) -``` - -The argument we are passing to `start_link` is the name of the process. It's common to give names to processes under supervision so that other processes can access them by name without needing to know their pid. This is useful because a supervised process might crash, in which case its pid will change when the supervisor restarts it. By using a name, we can guarantee the newly started process will register itself under the same name, without a need to explicitly fetch the latest pid. Notice it is also common to register the process under the same name of the module that defines it, this makes things more straight-forward when debugging or introspecting a live-system. - -Finally, we call `supervise/2`, passing the list of children and the strategy of `:one_for_one`. - -The supervision strategy dictates what happens when one of the children crashes. `:one_for_one` means that if a child dies, it will be the only one restarted. Since we have only one child now, that's all we need. The `Supervisor` behaviour supports many different strategies and we will discuss them in this chapter. - -Since `KV.Registry.start_link/1` is now expecting an argument, we need to change our implementation to receive such argument. Open up `lib/kv/registry.ex` and replace the `start_link/0` definition by: - -```elixir -@doc """ -Starts the registry with the given `name`. -""" -def start_link(name) do - GenServer.start_link(__MODULE__, :ok, name: name) -end -``` - -We also need to update our tests to give a name when starting the registry. Replace the `setup` function in `test/kv/registry_test.exs` by: - -```elixir -setup context do - {:ok, registry} = KV.Registry.start_link(context.test) - {:ok, registry: registry} -end -``` - -`setup/2` may also receive the test context, similar to `test/3`. Besides whatever value we may add in our setup blocks, the context includes some default keys, like `:case`, `:test`, `:file` and `:line`. We have used `context.test` as a shortcut to spawn a registry with the same name of the test currently running. - -Now with our tests passing, we can take our supervisor for a spin. If we start a console inside our project using `iex -S mix`, we can manually start the supervisor: - -```iex -iex> KV.Supervisor.start_link -{:ok, #PID<0.66.0>} -iex> KV.Registry.create(KV.Registry, "shopping") -:ok -iex> KV.Registry.lookup(KV.Registry, "shopping") -{:ok, #PID<0.70.0>} -``` - -When we started the supervisor, the registry worker was automatically started, allowing us to create buckets without the need to manually start it. - -In practice we rarely start the application supervisor manually. Instead it is started as part of the application callback. - -## Understanding applications - -We have been working inside an application this entire time. Every time we changed a file and ran `mix compile`, we could see a `Generated kv app` message in the compilation output. - -We can find the generated `.app` file at `_build/dev/lib/kv/ebin/kv.app`. Let's have a look at its contents: - -```erlang -{application,kv, - [{registered,[]}, - {description,"kv"}, - {applications,[kernel,stdlib,elixir,logger]}, - {vsn,"0.0.1"}, - {modules,['Elixir.KV','Elixir.KV.Bucket', - 'Elixir.KV.Registry','Elixir.KV.Supervisor']}]}. -``` - -This file contains Erlang terms (written using Erlang syntax). Even though we are not familiar with Erlang, it is easy to guess this file holds our application definition. It contains our application `version`, all the modules defined by it, as well as a list of applications we depend on, like Erlang's `kernel`, `elixir` itself, and `logger` which is specified in the application list in `mix.exs`. - -It would be pretty boring to update this file manually every time we add a new module to our application. That's why Mix generates and maintains it for us. - -We can also configure the generated `.app` file by customizing the values returned by the `application/0` inside our `mix.exs` project file. We are going to do our first customization soon. - -### Starting applications - -When we define a `.app` file, which is the application specification, we are able to start and stop the application as a whole. We haven't worried about this so far for two reasons: - -1. Mix automatically starts our current application for us - -2. Even if Mix didn't start our application for us, our application does not yet do anything when it starts - -In any case, let's see how Mix starts the application for us. Let's start a project console with `iex -S mix` and try: - -```iex -iex> Application.start(:kv) -{:error, {:already_started, :kv}} -``` - -Oops, it's already started. Mix normally starts the whole hierarchy of applications defined in our project's `mix.exs` file and it does the same for all dependencies if they depend on other applications. - -We can pass an option to Mix to ask it to not start our application. Let's give it a try by running `iex -S mix run --no-start`: - -```iex -iex> Application.start(:kv) -:ok -``` - -We can stop our `:kv` application as well as the `:logger` application, which is started by default with Elixir: - -```iex -iex> Application.stop(:kv) -:ok -iex> Application.stop(:logger) -:ok -``` - -And let's try to start our application again: - -```iex -iex> Application.start(:kv) -{:error, {:not_started, :logger}} -``` - -Now we get an error because an application that `:kv` depends on (`:logger` in this case) isn't started. We need to either start each application manually in the correct order or call `Application.ensure_all_started` as follows: - -```iex -iex> Application.ensure_all_started(:kv) -{:ok, [:logger, :kv]} -``` - -Nothing really exciting happens but it shows how we can control our application. - -> When you run `iex -S mix`, it is equivalent to running `iex -S mix run`. So whenever you need to pass more options to Mix when starting IEx, it's a matter of typing `iex -S mix run` and then passing any options the `run` command accepts. You can find more information about `run` by running `mix help run` in your shell. - -### The application callback - -Since we spent all this time talking about how applications are started and stopped, there must be a way to do something useful when the application starts. And indeed, there is! - -We can specify an application callback function. This is a function that will be invoked when the application starts. The function must return a result of `{:ok, pid}`, where `pid` is the process identifier of a supervisor process. - -We can configure the application callback in two steps. First, open up the `mix.exs` file and change `def application` to the following: - -```elixir -def application do - [applications: [:logger], - mod: {KV, []}] -end -``` - -The `:mod` option specifies the "application callback module", followed by the arguments to be passed on application start. The application callback module can be any module that implements the [Application](https://hexdocs.pm/elixir/Application.html) behaviour. - -Now that we have specified `KV` as the module callback, we need to change the `KV` module, defined in `lib/kv.ex`: - -```elixir -defmodule KV do - use Application - - def start(_type, _args) do - KV.Supervisor.start_link - end -end -``` - -When we `use Application`, we need to define a couple functions, similar to when we used `Supervisor` or `GenServer`. This time we only need to define a `start/2` function. If we wanted to specify custom behaviour on application stop, we could define a `stop/1` function. - -Let's start our project console once again with `iex -S mix`. We will see a process named `KV.Registry` is already running: - -```iex -iex> KV.Registry.create(KV.Registry, "shopping") -:ok -iex> KV.Registry.lookup(KV.Registry, "shopping") -{:ok, #PID<0.88.0>} -``` - -How do we know this is working? After all, we are creating the bucket and then looking it up; of course it should work, right? Well, remember that `KV.Registry.create/2` uses `GenServer.cast/2`, and therefore will return `:ok` regardless of whether the message finds its target or not. At that point, we don't know whether the supervisor and the server are up, and if the bucket was created. However, `KV.Registry.lookup/2` uses `GenServer.call/3`, and will block and wait for a response from the server. We do get a positive response, so we know all is up and running. - -For an experiment, try reimplementing `KV.Registry.create/2` to use `GenServer.call/3` instead, and momentarily disable the application callback. Run the code above on the console again, and you will see the creation step fail straightaway. - -Don't forget to bring the code back to normal before resuming this tutorial! - -### Projects or applications? - -Mix makes a distinction between projects and applications. Based on the contents of our `mix.exs` file, we would say we have a Mix project that defines the `:kv` application. As we will see in later chapters, there are projects that don't define any application. - -When we say "project" you should think about Mix. Mix is the tool that manages your project. It knows how to compile your project, test your project and more. It also knows how to compile and start the application relevant to your project. - -When we talk about applications, we talk about OTP. Applications are the entities that are started and stopped as a whole by the runtime. You can learn more about applications in the [docs for the Application module](https://hexdocs.pm/elixir/Application.html), as well as by running `mix help compile.app` to learn more about the supported options in `def application`. - -## Simple one for one supervisors - -We have now successfully defined our supervisor which is automatically started (and stopped) as part of our application lifecycle. - -Remember however that our `KV.Registry` is both linking and monitoring bucket processes in the `handle_cast/2` callback: - -```elixir -{:ok, pid} = KV.Bucket.start_link -ref = Process.monitor(pid) -``` - -Links are bi-directional, which implies that a crash in a bucket will crash the registry. Although we now have the supervisor, which guarantees the registry will be back up and running, crashing the registry still means we lose all data associating bucket names to their respective processes. - -In other words, we want the registry to keep on running even if a bucket crashes. Let's write a new registry test: - -```elixir -test "removes bucket on crash", %{registry: registry} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - - # Stop the bucket with non-normal reason - Process.exit(bucket, :shutdown) - - # Wait until the bucket is dead - ref = Process.monitor(bucket) - assert_receive {:DOWN, ^ref, _, _, _} - - assert KV.Registry.lookup(registry, "shopping") == :error -end -``` - -The test is similar to "removes bucket on exit" except that we are being a bit more harsh by sending `:shutdown` as the exit reason instead of `:normal`. Opposite to `Agent.stop/1`, `Process.exit/2` is an asynchronous operation, therefore we cannot simply query `KV.Registry.lookup/2` right after sending the exit signal because there will be no guarantee the bucket will be dead by then. To solve this, we also monitor the bucket during test and only query the registry once we are sure it is DOWN, avoiding race conditions. - -Since the bucket is linked to the registry, which is then linked to the test process, killing the bucket causes the registry to crash which then causes the test process to crash too: - -``` -1) test removes bucket on crash (KV.RegistryTest) - test/kv/registry_test.exs:52 - ** (EXIT from #PID<0.94.0>) shutdown -``` - -One possible solution to this issue would be to provide a `KV.Bucket.start/0`, that invokes `Agent.start/1`, and use it from the registry, removing the link between registry and buckets. However, this would be a bad idea because buckets would not be linked to any process after this change. This means that, if someone stops the `:kv` application, all buckets would remain alive as they are unreachable. Not only that, if a process is unreacheable, they are harder to introspect. - -We are going to solve this issue by defining a new supervisor that will spawn and supervise all buckets. There is one supervisor strategy, called `:simple_one_for_one`, that is the perfect fit for such situations: it allows us to specify a worker template and supervise many children based on this template. With this strategy, no workers are started during the supervisor initialization, and a new worker is started each time `start_child/2` is called. - -Let's define our `KV.Bucket.Supervisor` in `lib/kv/bucket/supervisor.ex` as follows: - -```elixir -defmodule KV.Bucket.Supervisor do - use Supervisor - - # A simple module attribute that stores the supervisor name - @name KV.Bucket.Supervisor - - def start_link do - Supervisor.start_link(__MODULE__, :ok, name: @name) - end - - def start_bucket do - Supervisor.start_child(@name, []) - end - - def init(:ok) do - children = [ - worker(KV.Bucket, [], restart: :temporary) - ] - - supervise(children, strategy: :simple_one_for_one) - end -end -``` - -There are three changes in this supervisor compared to the first one. - -First of all, we have decided to give the supervisor a local name of `KV.Bucket.Supervisor`. We have also defined a `start_bucket/0` function that will start a bucket as a child of our supervisor named `KV.Bucket.Supervisor`. `start_bucket/0` is the function we are going to invoke instead of calling `KV.Bucket.start_link` directly in the registry. - -Finally, in the `init/1` callback, we are marking the worker as `:temporary`. This means that if the bucket dies, it won't be restarted. That's because we only want to use the supervisor as a mechanism to group the buckets. - -Run `iex -S mix` so we can give our new supervisor a try: - -```iex -iex> {:ok, _} = KV.Bucket.Supervisor.start_link -{:ok, #PID<0.70.0>} -iex> {:ok, bucket} = KV.Bucket.Supervisor.start_bucket -{:ok, #PID<0.72.0>} -iex> KV.Bucket.put(bucket, "eggs", 3) -:ok -iex> KV.Bucket.get(bucket, "eggs") -3 -``` - -Let's change the registry to work with the buckets supervisor by rewriting how buckets are started: - -```elixir -def handle_cast({:create, name}, {names, refs}) do - if Map.has_key?(names, name) do - {:noreply, {names, refs}} - else - {:ok, pid} = KV.Bucket.Supervisor.start_bucket - ref = Process.monitor(pid) - refs = Map.put(refs, ref, name) - names = Map.put(names, name, pid) - {:noreply, {names, refs}} - end -end -``` - -Once we perform those changes, our test suite should fail as there is no bucket supervisor. Instead of directly starting the bucket supervisor on every test, let's automatically start it as part of our main supervision tree. - -## Supervision trees - -In order to use the buckets supervisor in our application, we need to add it as a child of `KV.Supervisor`. Notice we are beginning to have supervisors that supervise other supervisors, forming so-called "supervision trees". - -Open up `lib/kv/supervisor.ex` and change `init/1` to match the following: - -```elixir -def init(:ok) do - children = [ - worker(KV.Registry, [KV.Registry]), - supervisor(KV.Bucket.Supervisor, []) - ] - - supervise(children, strategy: :one_for_one) -end -``` - -This time we have added a supervisor as child, starting it with no arguments. Re-run the test suite and now all tests should pass. - -Since we have added more children to the supervisor, it is also important to evaluate if the `:one_for_one` supervision strategy is still correct. One flaw that shows up right away is the relationship between the `KV.Registry` worker process and the `KV.Bucket.Supervisor` supervisor process. If `KV.Registry` dies, all information linking `KV.Bucket` names to `KV.Bucket` processes is lost, and therefore `KV.Bucket.Supervisor` must die too- otherwise, the `KV.Bucket` processes it manages would be orphaned. - -In light of this observation, we should consider moving to another supervision strategy. The two other candidates are `:one_for_all` and `:rest_for_one`. A supervisor using the `:one_for_all` strategy will kill and restart all of its children processes whenever any one of them dies. At first glance, this would appear to suit our use case, but it also seems a little heavy-handed, because `KV.Registry` is perfectly capable of cleaning itself up if `KV.Bucket.Supervisor` dies. In this case, the `:rest_for_one` strategy comes in handy: when a child process crashes, the supervisor will only kill and restart child processes which were started *after* the crashed child. Let's rewrite our supervision tree to use this strategy instead: - -```elixir -def init(:ok) do - children = [ - worker(KV.Registry, [KV.Registry]), - supervisor(KV.Bucket.Supervisor, []) - ] - - supervise(children, strategy: :rest_for_one) -end -``` - -Now, if the registry worker crashes, both the registry and the "rest" of `KV.Supervisor`'s children (i.e. `KV.Bucket.Supervisor`) will be restarted. However, if `KV.Bucket.Supervisor` crashes, `KV.Registry` will not be restarted, because it was started prior to `KV.Bucket.Supervisor`. - -There are other strategies and other options that could be given to `worker/2`, `supervisor/2` and `supervise/2` functions, so don't forget to check both [`Supervisor`](https://hexdocs.pm/elixir/Supervisor.html) and [`Supervisor.Spec`](https://hexdocs.pm/elixir/Supervisor.Spec.html) modules. - -To help developers remember how to work with Supervisors and its convenience functions, [Benjamin Tan Wei Hao](http://benjamintan.io/) has created a [Supervisor cheat sheet](https://raw.githubusercontent.com/benjamintanweihao/elixir-cheatsheets/master/Supervisor_CheatSheet.pdf). - -There are two topics left before we move on to the next chapter. - -## Observer - -Now that we have defined our supervision tree, it is a great opportunity to introduce the Observer tool that ships with Erlang. Start your application with `iex -S mix` and key this in: - -```iex -iex> :observer.start -``` - -A GUI should pop-up containing all sorts of information about our system, from general statistics to load charts as well as a list of all running processes and applications. - -In the Applications tab, you will see all applications currently running in your system along side their supervision tree. You can select the `kv` application to explore it further: - - - -Not only that, as you create new buckets on the terminal, you should see new processes spawned in the supervision tree shown in Observer: - -```iex -iex> KV.Registry.create KV.Registry, "shopping" -:ok -``` - -We will leave it up to you to further explore what Observer provides. Note you can double click any process in the supervision tree to retrieve more information about it, as well as right-click a process to send "a kill signal", a perfect way to emulate failures and see if your supervisor reacts as expected. - -At the end of the day, tools like Observer is one of the main reasons you want to always start processes inside supervision trees, even if they are temporary, to ensure they are always reachable and introspectable. - -## Shared state in tests - -So far we have been starting one registry per test to ensure they are isolated: - -```elixir -setup context do - {:ok, registry} = KV.Registry.start_link(context.test) - {:ok, registry: registry} -end -``` - -Since we have now changed our registry to use `KV.Bucket.Supervisor`, which is registered globally, our tests are now relying on this shared, global supervisor even though each test has its own registry. The question is: should we? - -It depends. It is ok to rely on shared global state as long as we depend only on a non-shared partition of this state. For example, every time we register a process under a given name, we are registering a process against a shared name registry. However, as long as we guarantee the names are specific to each test, by using a construct like `context.test`, we won't have concurrency or data dependency issues between tests. - -Similar reasoning should be applied to our bucket supervisor. Although multiple registries may start buckets on the shared bucket supervisor, those buckets and registries are isolated from each other. We would only run into concurrency issues if we used a function like `Supervisor.count_children(KV.Bucket.Supervisor)` which would count all buckets from all registries, potentially giving different results when tests run concurrently. - -Since we have relied only on a non-shared partition of the bucket supervisor so far, we don't need to worry about concurrency issues in our test suite. In case it ever becomes a problem, we can start a supervisor per test and pass it as an argument to the registry `start_link` function. - -Now that our application is properly supervised and tested, let's see how we can speed things up. diff --git a/getting-started/mix-otp/task-and-gen-tcp.markdown b/getting-started/mix-otp/task-and-gen-tcp.markdown index 5fec7485c..a278081ed 100644 --- a/getting-started/mix-otp/task-and-gen-tcp.markdown +++ b/getting-started/mix-otp/task-and-gen-tcp.markdown @@ -1,298 +1,5 @@ --- -layout: getting-started -title: Task and gen_tcp +layout: redirect +sitemap: false +redirect_to: task-and-gen-tcp --- - -# {{ page.title }} - -{% include toc.html %} - -{% include mix-otp-preface.html %} - -In this chapter, we are going to learn how to use [Erlang's `:gen_tcp` module](http://www.erlang.org/doc/man/gen_tcp.html) to serve requests. This provides a great opportunity to explore Elixir's `Task` module. In future chapters we will expand our server so it can actually serve the commands. - -## Echo server - -We will start our TCP server by first implementing an echo server. It will send a response with the text it received in the request. We will slowly improve our server until it is supervised and ready to handle multiple connections. - -A TCP server, in broad strokes, performs the following steps: - -1. Listens to a port until the port is available and it gets hold of the socket -2. Waits for a client connection on that port and accepts it -3. Reads the client request and writes a response back - -Let's implement those steps. Move to the `apps/kv_server` application, open up `lib/kv_server.ex`, and add the following functions: - -```elixir -require Logger - -def accept(port) do - # The options below mean: - # - # 1. `:binary` - receives data as binaries (instead of lists) - # 2. `packet: :line` - receives data line by line - # 3. `active: false` - blocks on `:gen_tcp.recv/2` until data is available - # 4. `reuseaddr: true` - allows us to reuse the address if the listener crashes - # - {:ok, socket} = :gen_tcp.listen(port, - [:binary, packet: :line, active: false, reuseaddr: true]) - Logger.info "Accepting connections on port #{port}" - loop_acceptor(socket) -end - -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - serve(client) - loop_acceptor(socket) -end - -defp serve(socket) do - socket - |> read_line() - |> write_line(socket) - - serve(socket) -end - -defp read_line(socket) do - {:ok, data} = :gen_tcp.recv(socket, 0) - data -end - -defp write_line(line, socket) do - :gen_tcp.send(socket, line) -end -``` - -We are going to start our server by calling `KVServer.accept(4040)`, where 4040 is the port. The first step in `accept/1` is to listen to the port until the socket becomes available and then call `loop_acceptor/1`. `loop_acceptor/1` is a loop accepting client connections. For each accepted connection, we call `serve/1`. - -`serve/1` is another loop that reads a line from the socket and writes those lines back to the socket. Note that the `serve/1` function uses [the pipe operator `|>`](https://hexdocs.pm/elixir/Kernel.html#%7C%3E/2) to express this flow of operations. The pipe operator evaluates the left side and passes its result as first argument to the function on the right side. The example above: - -```elixir -socket |> read_line() |> write_line(socket) -``` - -is equivalent to: - -```elixir -write_line(read_line(socket), socket) -``` - -The `read_line/1` implementation receives data from the socket using `:gen_tcp.recv/2` and `write_line/2` writes to the socket using `:gen_tcp.send/2`. - -Note that `serve/1` is an infinite loop called sequentially inside `loop_acceptor/1`, so the tail call to `loop_acceptor/1` is never reached and could be avoided. However, as we shall see, we will need to execute `serve/1` in a separate process, so we will need that tail call soon. - -This is pretty much all we need to implement our echo server. Let's give it a try! - -Start an IEx session inside the `kv_server` application with `iex -S mix`. Inside IEx, run: - -```iex -iex> KVServer.accept(4040) -``` - -The server is now running, and you will even notice the console is blocked. Let's use [a `telnet` client](https://en.wikipedia.org/wiki/Telnet) to access our server. There are clients available on most operating systems, and their command lines are generally similar: - -```bash -$ telnet 127.0.0.1 4040 -Trying 127.0.0.1... -Connected to localhost. -Escape character is '^]'. -hello -hello -is it me -is it me -you are looking for? -you are looking for? -``` - -Type "hello", press enter, and you will get "hello" back. Excellent! - -My particular telnet client can be exited by typing `ctrl + ]`, typing `quit`, and pressing ``, but your client may require different steps. - -Once you exit the telnet client, you will likely see an error in the IEx session: - - ** (MatchError) no match of right hand side value: {:error, :closed} - (kv_server) lib/kv_server.ex:41: KVServer.read_line/1 - (kv_server) lib/kv_server.ex:33: KVServer.serve/1 - (kv_server) lib/kv_server.ex:27: KVServer.loop_acceptor/1 - -That's because we were expecting data from `:gen_tcp.recv/2` but the client closed the connection. We need to handle such cases better in future revisions of our server. - -For now there is a more important bug we need to fix: what happens if our TCP acceptor crashes? Since there is no supervision, the server dies and we won't be able to serve more requests, because it won't be restarted. That's why we must move our server to a supervision tree. - -## Tasks - -We have learned about agents, generic servers, and supervisors. They are all meant to work with multiple messages or manage state. But what do we use when we only need to execute some task and that is it? - -[The Task module](https://hexdocs.pm/elixir/Task.html) provides this functionality exactly. For example, it has a `start_link/3` function that receives a module, function and arguments, allowing us to run a given function as part of a supervision tree. - -Let's give it a try. Open up `lib/kv_server.ex`, and let's change the supervisor in the `start/2` function to the following: - -```elixir -def start(_type, _args) do - import Supervisor.Spec - - children = [ - worker(Task, [KVServer, :accept, [4040]]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) -end -``` - -With this change, we are saying that we want to run `KVServer.accept(4040)` as a worker. We are hardcoding the port for now, but we will discuss ways in which this could be changed later. - -Now that the server is part of the supervision tree, it should start automatically when we run the application. Type `mix run --no-halt` in the terminal, and once again use the `telnet` client to make sure that everything still works: - -```bash -$ telnet 127.0.0.1 4040 -Trying 127.0.0.1... -Connected to localhost. -Escape character is '^]'. -say you -say you -say me -say me -``` - -Yes, it works! If you kill the client, causing the whole server to crash, you will see another one starts right away. However, does it *scale*? - -Try to connect two telnet clients at the same time. When you do so, you will notice that the second client doesn't echo: - -```bash -$ telnet 127.0.0.1 4040 -Trying 127.0.0.1... -Connected to localhost. -Escape character is '^]'. -hello -hello? -HELLOOOOOO? -``` - -It doesn't seem to work at all. That's because we are serving requests in the same process that are accepting connections. When one client is connected, we can't accept another client. - -## Task supervisor - -In order to make our server handle simultaneous connections, we need to have one process working as an acceptor that spawns other processes to serve requests. One solution would be to change: - -```elixir -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - serve(client) - loop_acceptor(socket) -end -``` - -to use `Task.start_link/1`, which is similar to `Task.start_link/3`, but it receives an anonymous function instead of module, function and arguments: - -```elixir -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - Task.start_link(fn -> serve(client) end) - loop_acceptor(socket) -end -``` - -We are starting a linked Task directly from the acceptor process. But we've already made this mistake once. Do you remember? - -This is similar to the mistake we made when we called `KV.Bucket.start_link/0` straight from the registry. That meant a failure in any bucket would bring the whole registry down. - -The code above would have the same flaw: if we link the `serve(client)` task to the acceptor, a crash when serving a request would bring the acceptor, and consequently all other connections, down. - -We fixed the issue for the registry by using a simple one for one supervisor. We are going to use the same tactic here, except that this pattern is so common with tasks that `Task` already comes with a solution: a simple one for one supervisor that starts temporary tasks as part of our supervision tree! - -Let's change `start/2` once again, to add a supervisor to our tree: - -```elixir -def start(_type, _args) do - import Supervisor.Spec - - children = [ - supervisor(Task.Supervisor, [[name: KVServer.TaskSupervisor]]), - worker(Task, [KVServer, :accept, [4040]]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) -end -``` - -We'll now start a [`Task.Supervisor`](https://hexdocs.pm/elixir/Task.Supervisor.html) process with name `KVServer.TaskSupervisor`. Remember, since the acceptor task depends on this supervisor, the supervisor must be started first. - -Now we need to change `loop_acceptor/1` to use `Task.Supervisor` to serve each request: - -```elixir -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - {:ok, pid} = Task.Supervisor.start_child(KVServer.TaskSupervisor, fn -> serve(client) end) - :ok = :gen_tcp.controlling_process(client, pid) - loop_acceptor(socket) -end -``` - -You might notice that we added a line, `:ok = :gen_tcp.controlling_process(client, pid)`. This makes the child process the "controlling process" of the `client` socket. If we didn't do this, the acceptor would bring down all the clients if it crashed because sockets would be tied to the process that accepted them (which is the default behaviour). - -Start a new server with `mix run --no-halt` and we can now open up many concurrent telnet clients. You will also notice that quitting a client does not bring the acceptor down. Excellent! - -Here is the full echo server implementation, in a single module: - -```elixir -defmodule KVServer do - use Application - require Logger - - @doc false - def start(_type, _args) do - import Supervisor.Spec - - children = [ - supervisor(Task.Supervisor, [[name: KVServer.TaskSupervisor]]), - worker(Task, [KVServer, :accept, [4040]]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) - end - - @doc """ - Starts accepting connections on the given `port`. - """ - def accept(port) do - {:ok, socket} = :gen_tcp.listen(port, - [:binary, packet: :line, active: false, reuseaddr: true]) - Logger.info "Accepting connections on port #{port}" - loop_acceptor(socket) - end - - defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - {:ok, pid} = Task.Supervisor.start_child(KVServer.TaskSupervisor, fn -> serve(client) end) - :ok = :gen_tcp.controlling_process(client, pid) - loop_acceptor(socket) - end - - defp serve(socket) do - socket - |> read_line() - |> write_line(socket) - - serve(socket) - end - - defp read_line(socket) do - {:ok, data} = :gen_tcp.recv(socket, 0) - data - end - - defp write_line(line, socket) do - :gen_tcp.send(socket, line) - end -end -``` - -Since we have changed the supervisor specification, we need to ask: is our supervision strategy still correct? - -In this case, the answer is yes: if the acceptor crashes, there is no need to crash the existing connections. On the other hand, if the task supervisor crashes, there is no need to crash the acceptor too. - -In the next chapter we will start parsing the client requests and sending responses, finishing our server. diff --git a/getting-started/module-attributes.markdown b/getting-started/module-attributes.markdown index 65e317f02..026b62136 100644 --- a/getting-started/module-attributes.markdown +++ b/getting-started/module-attributes.markdown @@ -1,165 +1,5 @@ --- -layout: getting-started -title: Module attributes +layout: redirect +sitemap: false +redirect_to: module-attributes --- - -# {{ page.title }} - -{% include toc.html %} - -Module attributes in Elixir serve three purposes: - -1. They serve to annotate the module, often with information to be used by the user or the VM. -2. They work as constants. -3. They work as a temporary module storage to be used during compilation. - -Let's check each case, one by one. - -## As annotations - -Elixir brings the concept of module attributes from Erlang. For example: - -```elixir -defmodule MyServer do - @vsn 2 -end -``` - -In the example above, we are explicitly setting the version attribute for that module. `@vsn` is used by the code reloading mechanism in the Erlang VM to check if a module has been updated or not. If no version is specified, the version is set to the MD5 checksum of the module functions. - -Elixir has a handful of reserved attributes. Here are a few of them, the most commonly used ones: - -* `@moduledoc` - provides documentation for the current module. -* `@doc` - provides documentation for the function or macro that follows the attribute. -* `@behaviour` - (notice the British spelling) used for specifying an OTP or user-defined behaviour. -* `@before_compile` - provides a hook that will be invoked before the module is compiled. This makes it possible to inject functions inside the module exactly before compilation. - -`@moduledoc` and `@doc` are by far the most used attributes, and we expect you to use them a lot. Elixir treats documentation as first-class and provides many functions to access documentation. You can read more about [writing documentation in Elixir in our official documentation](https://hexdocs.pm/elixir/writing-documentation.html). - -Let's go back to the `Math` module defined in the previous chapters, add some documentation and save it to the `math.ex` file: - -```elixir -defmodule Math do - @moduledoc """ - Provides math-related functions. - - ## Examples - - iex> Math.sum(1, 2) - 3 - - """ - - @doc """ - Calculates the sum of two numbers. - """ - def sum(a, b), do: a + b -end -``` - -Elixir promotes the use of markdown with heredocs to write readable documentation. Heredocs are multiline strings, they start and end with triple double-quotes, keeping the formatting of the inner text. We can access the documentation of any compiled module directly from IEx: - -```bash -$ elixirc math.ex -$ iex -``` - -```iex -iex> h Math # Access the docs for the module Math -... -iex> h Math.sum # Access the docs for the sum function -... -``` - -We also provide a tool called [ExDoc](https://github.com/elixir-lang/ex_doc) which is used to generate HTML pages from the documentation. - -You can take a look at the docs for [Module](https://hexdocs.pm/elixir/Module.html) for a complete list of supported attributes. Elixir also uses attributes to define [typespecs](/getting-started/typespecs-and-behaviours.html). - -This section covers built-in attributes. However, attributes can also be used by developers or extended by libraries to support custom behaviour. - -## As constants - -Elixir developers will often use module attributes as constants: - -```elixir -defmodule MyServer do - @initial_state %{host: "147.0.0.1", port: 3456} - IO.inspect @initial_state -end -``` - -> Note: Unlike Erlang, user defined attributes are not stored in the module by default. The value exists only during compilation time. A developer can configure an attribute to behave closer to Erlang by calling [`Module.register_attribute/3`](https://hexdocs.pm/elixir/Module.html#register_attribute/3). - -Trying to access an attribute that was not defined will print a warning: - -```elixir -defmodule MyServer do - @unknown -end -warning: undefined module attribute @unknown, please remove access to @unknown or explicitly set it before access -``` - -Finally, attributes can also be read inside functions: - -```elixir -defmodule MyServer do - @my_data 14 - def first_data, do: @my_data - @my_data 13 - def second_data, do: @my_data -end - -MyServer.first_data #=> 14 -MyServer.second_data #=> 13 -``` - -Every time an attribute is read inside a function, a snapshot of its current value is taken. In other words, the value is read at compilation time and not at runtime. As we are going to see, this also makes attributes useful to be used as storage during module compilation. - -## As temporary storage - -One of the projects in the Elixir organization is [the `Plug` project](https://github.com/elixir-lang/plug), which is meant to be a common foundation for building web libraries and frameworks in Elixir. - -The Plug library also allows developers to define their own plugs which can be run in a web server: - -```elixir -defmodule MyPlug do - use Plug.Builder - - plug :set_header - plug :send_ok - - def set_header(conn, _opts) do - put_resp_header(conn, "x-header", "set") - end - - def send_ok(conn, _opts) do - send(conn, 200, "ok") - end -end - -IO.puts "Running MyPlug with Cowboy on http://localhost:4000" -Plug.Adapters.Cowboy.http MyPlug, [] -``` - -In the example above, we have used the `plug/1` macro to connect functions that will be invoked when there is a web request. Internally, every time you call `plug/1`, the Plug library stores the given argument in a `@plugs` attribute. Just before the module is compiled, Plug runs a callback that defines a function (`call/2`) which handles http requests. This function will run all plugs inside `@plugs` in order. - -In order to understand the underlying code, we'd need macros, so we will revisit this pattern in the meta-programming guide. However the focus here is on how using module attributes as storage allows developers to create DSLs. - -Another example comes from [the ExUnit framework](https://hexdocs.pm/ex_unit/) which uses module attributes as annotation and storage: - -```elixir -defmodule MyTest do - use ExUnit.Case - - @tag :external - test "contacts external service" do - # ... - end -end -``` - -Tags in ExUnit are used to annotate tests. Tags can be later used to filter tests. For example, you can avoid running external tests on your machine because they are slow and dependent on other services, while they can still be enabled in your build system. - -We hope this section shines some light on how Elixir supports meta-programming and how module attributes play an important role when doing so. - -In the next chapters we'll explore structs and protocols before moving to exception handling and other constructs like sigils and comprehensions. diff --git a/getting-started/modules-and-functions.markdown b/getting-started/modules-and-functions.markdown new file mode 100644 index 000000000..263315f14 --- /dev/null +++ b/getting-started/modules-and-functions.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: modules-and-functions +--- diff --git a/getting-started/modules.markdown b/getting-started/modules.markdown index 8425006e1..263315f14 100644 --- a/getting-started/modules.markdown +++ b/getting-started/modules.markdown @@ -1,284 +1,5 @@ --- -layout: getting-started -title: Modules +layout: redirect +sitemap: false +redirect_to: modules-and-functions --- - -# {{ page.title }} - -{% include toc.html %} - -In Elixir we group several functions into modules. We've already used many different modules in the previous chapters such as [the `String` module](https://hexdocs.pm/elixir/String.html): - -```iex -iex> String.length("hello") -5 -``` - -In order to create our own modules in Elixir, we use the `defmodule` macro. We use the `def` macro to define functions in that module: - -```iex -iex> defmodule Math do -...> def sum(a, b) do -...> a + b -...> end -...> end - -iex> Math.sum(1, 2) -3 -``` - -In the following sections, our examples are going to get longer in size, and it can be tricky to type them all in the shell. It's about time for us to learn how to compile Elixir code and also how to run Elixir scripts. - -## Compilation - -Most of the time it is convenient to write modules into files so they can be compiled and reused. Let's assume we have a file named `math.ex` with the following contents: - -```elixir -defmodule Math do - def sum(a, b) do - a + b - end -end -``` - -This file can be compiled using `elixirc`: - -```bash -$ elixirc math.ex -``` - -This will generate a file named `Elixir.Math.beam` containing the bytecode for the defined module. If we start `iex` again, our module definition will be available (provided that `iex` is started in the same directory the bytecode file is in): - -```iex -iex> Math.sum(1, 2) -3 -``` - -Elixir projects are usually organized into three directories: - -* ebin - contains the compiled bytecode -* lib - contains elixir code (usually `.ex` files) -* test - contains tests (usually `.exs` files) - -When working on actual projects, the build tool called `mix` will be responsible for compiling and setting up the proper paths for you. For learning purposes, Elixir also supports a scripted mode which is more flexible and does not generate any compiled artifacts. - -## Scripted mode - -In addition to the Elixir file extension `.ex`, Elixir also supports `.exs` files for scripting. Elixir treats both files exactly the same way, the only difference is in intention. `.ex` files are meant to be compiled while `.exs` files are used for scripting. When executed, both extensions compile and load their modules into memory, although only `.ex` files write their bytecode to disk in the format of `.beam` files. - -For instance, we can create a file called `math.exs`: - -```elixir -defmodule Math do - def sum(a, b) do - a + b - end -end - -IO.puts Math.sum(1, 2) -``` - -And execute it as: - -```bash -$ elixir math.exs -``` - -The file will be compiled in memory and executed, printing "3" as the result. No bytecode file will be created. In the following examples, we recommend you write your code into script files and execute them as shown above. - -## Named functions - -Inside a module, we can define functions with `def/2` and private functions with `defp/2`. A function defined with `def/2` can be invoked from other modules while a private function can only be invoked locally. - -```elixir -defmodule Math do - def sum(a, b) do - do_sum(a, b) - end - - defp do_sum(a, b) do - a + b - end -end - -IO.puts Math.sum(1, 2) #=> 3 -IO.puts Math.do_sum(1, 2) #=> ** (UndefinedFunctionError) -``` - -Function declarations also support guards and multiple clauses. If a function has several clauses, Elixir will try each clause until it finds one that matches. Here is an implementation of a function that checks if the given number is zero or not: - -```elixir -defmodule Math do - def zero?(0) do - true - end - - def zero?(x) when is_integer(x) do - false - end -end - -IO.puts Math.zero?(0) #=> true -IO.puts Math.zero?(1) #=> false -IO.puts Math.zero?([1, 2, 3]) #=> ** (FunctionClauseError) -IO.puts Math.zero?(0.0) #=> ** (FunctionClauseError) -``` - -Giving an argument that does not match any of the clauses raises an error. - -Similar to constructs like `if`, named functions support both `do:` and `do`/`end` block syntax, as [we learned `do`/`end` is a convenient syntax for the keyword list format](/getting-started/case-cond-and-if.html#doend-blocks). For example, we can edit `math.exs` to look like this: - -```elixir -defmodule Math do - def zero?(0), do: true - def zero?(x) when is_integer(x), do: false -end -``` - -And it will provide the same behaviour. You may use `do:` for one-liners but always use `do`/`end` for functions spanning multiple lines. - -## Function capturing - -Throughout this tutorial, we have been using the notation `name/arity` to refer to functions. It happens that this notation can actually be used to retrieve a named function as a function type. Start `iex`, running the `math.exs` file defined above: - -```bash -$ iex math.exs -``` - -```iex -iex> Math.zero?(0) -true -iex> fun = &Math.zero?/1 -&Math.zero?/1 -iex> is_function(fun) -true -iex> fun.(0) -true -```` - -Local or imported functions, like `is_function/1`, can be captured without the module: - -```iex -iex> &is_function/1 -&:erlang.is_function/1 -iex> (&is_function/1).(fun) -true -``` - -Note the capture syntax can also be used as a shortcut for creating functions: - -```iex -iex> fun = &(&1 + 1) -#Function<6.71889879/1 in :erl_eval.expr/5> -iex> fun.(1) -2 -``` - -The `&1` represents the first argument passed into the function. `&(&1+1)` above is exactly the same as `fn x -> x + 1 end`. The syntax above is useful for short function definitions. - -If you want to capture a function from a module, you can do `&Module.function()`: - -```iex -iex> fun = &List.flatten(&1, &2) -&List.flatten/2 -iex> fun.([1, [[2], 3]], [4, 5]) -[1, 2, 3, 4, 5] -``` - -`&List.flatten(&1, &2)` is the same as writing `fn(list, tail) -> List.flatten(list, tail) end` which in this case is equivalent to `&List.flatten/2`. You can read more about the capture operator `&` in [the `Kernel.SpecialForms` documentation](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#&/1). - -## Default arguments - -Named functions in Elixir also support default arguments: - -```elixir -defmodule Concat do - def join(a, b, sep \\ " ") do - a <> sep <> b - end -end - -IO.puts Concat.join("Hello", "world") #=> Hello world -IO.puts Concat.join("Hello", "world", "_") #=> Hello_world -``` - -Any expression is allowed to serve as a default value, but it won't be evaluated during the function definition. Every time the function is invoked and any of its default values have to be used, the expression for that default value will be evaluated: - -```elixir -defmodule DefaultTest do - def dowork(x \\ IO.puts "hello") do - x - end -end -``` - -```iex -iex> DefaultTest.dowork -hello -:ok -iex> DefaultTest.dowork 123 -123 -iex> DefaultTest.dowork -hello -:ok -``` - -If a function with default values has multiple clauses, it is required to create a function head (without an actual body) for declaring defaults: - -```elixir -defmodule Concat do - def join(a, b \\ nil, sep \\ " ") - - def join(a, b, _sep) when is_nil(b) do - a - end - - def join(a, b, sep) do - a <> sep <> b - end -end - -IO.puts Concat.join("Hello", "world") #=> Hello world -IO.puts Concat.join("Hello", "world", "_") #=> Hello_world -IO.puts Concat.join("Hello") #=> Hello -``` - -When using default values, one must be careful to avoid overlapping function definitions. Consider the following example: - -```elixir -defmodule Concat do - def join(a, b) do - IO.puts "***First join" - a <> b - end - - def join(a, b, sep \\ " ") do - IO.puts "***Second join" - a <> sep <> b - end -end -``` - -If we save the code above in a file named "concat.ex" and compile it, Elixir will emit the following warning: - - warning: this clause cannot match because a previous clause at line 2 always matches - -The compiler is telling us that invoking the `join` function with two arguments will always choose the first definition of `join` whereas the second one will only be invoked when three arguments are passed: - -```bash -$ iex concat.exs -``` - -```iex -iex> Concat.join "Hello", "world" -***First join -"Helloworld" -``` - -```iex -iex> Concat.join "Hello", "world", "_" -***Second join -"Hello_world" -``` - -This finishes our short introduction to modules. In the next chapters, we will learn how to use named functions for recursion, explore Elixir lexical directives that can be used for importing functions from other modules and discuss module attributes. diff --git a/getting-started/optional-syntax.markdown b/getting-started/optional-syntax.markdown new file mode 100644 index 000000000..9231aebf5 --- /dev/null +++ b/getting-started/optional-syntax.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: optional-syntax +--- diff --git a/getting-started/pattern-matching.markdown b/getting-started/pattern-matching.markdown index 8b9672cae..bfa53ced1 100644 --- a/getting-started/pattern-matching.markdown +++ b/getting-started/pattern-matching.markdown @@ -1,185 +1,5 @@ --- -layout: getting-started -title: Pattern matching +layout: redirect +sitemap: false +redirect_to: pattern-matching --- - -# {{ page.title }} - -{% include toc.html %} - -In this chapter, we will show how the `=` operator in Elixir is actually a match operator and how to use it to pattern match inside data structures. Finally, we will learn about the pin operator `^` used to access previously bound values. - -## The match operator - -We have used the `=` operator a couple times to assign variables in Elixir: - -```iex -iex> x = 1 -1 -iex> x -1 -``` - -In Elixir, the `=` operator is actually called *the match operator*. Let's see why: - -```iex -iex> 1 = x -1 -iex> 2 = x -** (MatchError) no match of right hand side value: 1 -``` - -Notice that `1 = x` is a valid expression, and it matched because both the left and right side are equal to 1. When the sides do not match, a `MatchError` is raised. - -A variable can only be assigned on the left side of `=`: - -```iex -iex> 1 = unknown -** (CompileError) iex:1: undefined function unknown/0 -```` - -Since there is no variable `unknown` previously defined, Elixir imagined you were trying to call a function named `unknown/0`, but such a function does not exist. - -## Pattern matching - -The match operator is not only used to match against simple values, but it is also useful for destructuring more complex data types. For example, we can pattern match on tuples: - -```iex -iex> {a, b, c} = {:hello, "world", 42} -{:hello, "world", 42} -iex> a -:hello -iex> b -"world" -``` - -A pattern match will error if the sides can't be matched, for example if the tuples have different sizes: - -```iex -iex> {a, b, c} = {:hello, "world"} -** (MatchError) no match of right hand side value: {:hello, "world"} -``` - -And also when comparing different types: - -```iex -iex> {a, b, c} = [:hello, "world", 42] -** (MatchError) no match of right hand side value: [:hello, "world", 42] -``` - -More interestingly, we can match on specific values. The example below asserts that the left side will only match the right side when the right side is a tuple that starts with the atom `:ok`: - -```iex -iex> {:ok, result} = {:ok, 13} -{:ok, 13} -iex> result -13 - -iex> {:ok, result} = {:error, :oops} -** (MatchError) no match of right hand side value: {:error, :oops} -``` - -We can pattern match on lists: - -```iex -iex> [a, b, c] = [1, 2, 3] -[1, 2, 3] -iex> a -1 -``` - -A list also supports matching on its own head and tail: - -```iex -iex> [head | tail] = [1, 2, 3] -[1, 2, 3] -iex> head -1 -iex> tail -[2, 3] -``` - -Similar to the `hd/1` and `tl/1` functions, we can't match an empty list with a head and tail pattern: - -```iex -iex> [h | t] = [] -** (MatchError) no match of right hand side value: [] -``` - -The `[head | tail]` format is not only used on pattern matching but also for prepending items to a list: - -```iex -iex> list = [1, 2, 3] -[1, 2, 3] -iex> [0 | list] -[0, 1, 2, 3] -``` - -Pattern matching allows developers to easily destructure data types such as tuples and lists. As we will see in the following chapters, it is one of the foundations of recursion in Elixir and applies to other types as well, like maps and binaries. - -## The pin operator - -Variables in Elixir can be rebound: - -```iex -iex> x = 1 -1 -iex> x = 2 -2 -``` - -Use the pin operator `^` when you want to pattern match against an existing variable's value rather than rebinding the variable: - -```iex -iex> x = 1 -1 -iex> ^x = 2 -** (MatchError) no match of right hand side value: 2 -iex> {y, ^x} = {2, 1} -{2, 1} -iex> y -2 -iex> {y, ^x} = {2, 2} -** (MatchError) no match of right hand side value: {2, 2} -``` - -Because we have assigned the value of 1 to the variable x, this last example could also have been written as: - -``` -iex> {y, 1} = {2, 2} -** (MatchError) no match of right hand side value: {2, 2} -``` - -If a variable is mentioned more than once in a pattern, all references should bind to the same pattern: - -```iex -iex> {x, x} = {1, 1} -{1, 1} -iex> {x, x} = {1, 2} -** (MatchError) no match of right hand side value: {1, 2} -``` - -In some cases, you don't care about a particular value in a pattern. It is a common practice to bind those values to the underscore, `_`. For example, if only the head of the list matters to us, we can assign the tail to underscore: - -```iex -iex> [h | _] = [1, 2, 3] -[1, 2, 3] -iex> h -1 -``` - -The variable `_` is special in that it can never be read from. Trying to read from it gives an unbound variable error: - -```iex -iex> _ -** (CompileError) iex:1: unbound variable _ -``` - -Although pattern matching allows us to build powerful constructs, its usage is limited. For instance, you cannot make function calls on the left side of a match. The following example is invalid: - -```iex -iex> length([1, [2], 3]) = 3 -** (CompileError) iex:1: illegal pattern -``` - -This finishes our introduction to pattern matching. As we will see in the next chapter, pattern matching is very common in many language constructs. diff --git a/getting-started/processes.markdown b/getting-started/processes.markdown index 31c179ac7..1435024b8 100644 --- a/getting-started/processes.markdown +++ b/getting-started/processes.markdown @@ -1,249 +1,5 @@ --- -layout: getting-started -title: Processes +layout: redirect +sitemap: false +redirect_to: processes --- - -# {{ page.title }} - -{% include toc.html %} - -In Elixir, all code runs inside processes. Processes are isolated from each other, run concurrent to one another and communicate via message passing. Processes are not only the basis for concurrency in Elixir, but they also provide the means for building distributed and fault-tolerant programs. - -Elixir's processes should not be confused with operating system processes. Processes in Elixir are extremely lightweight in terms of memory and CPU (unlike threads in many other programming languages). Because of this, it is not uncommon to have tens or even hundreds of thousands of processes running simultaneously. - -In this chapter, we will learn about the basic constructs for spawning new processes, as well as sending and receiving messages between different processes. - -## `spawn` - -The basic mechanism for spawning new processes is with the auto-imported `spawn/1` function: - -```iex -iex> spawn fn -> 1 + 2 end -#PID<0.43.0> -``` - -`spawn/1` takes a function which it will execute in another process. - -Notice `spawn/1` returns a PID (process identifier). At this point, the process you spawned is very likely dead. The spawned process will execute the given function and exit after the function is done: - -```iex -iex> pid = spawn fn -> 1 + 2 end -#PID<0.44.0> -iex> Process.alive?(pid) -false -``` - -> Note: you will likely get different process identifiers than the ones we are getting in this guide. - -We can retrieve the PID of the current process by calling `self/0`: - -```iex -iex> self() -#PID<0.41.0> -iex> Process.alive?(self()) -true -``` - -Processes get much more interesting when we are able to send and receive messages. - -## `send` and `receive` - -We can send messages to a process with `send/2` and receive them with `receive/1`: - -```iex -iex> send self(), {:hello, "world"} -{:hello, "world"} -iex> receive do -...> {:hello, msg} -> msg -...> {:world, msg} -> "won't match" -...> end -"world" -``` - -When a message is sent to a process, the message is stored in the process mailbox. The `receive/1` block goes through the current process mailbox searching for a message that matches any of the given patterns. `receive/1` supports guards and many clauses, such as `case/2`. - -If there is no message in the mailbox matching any of the patterns, the current process will wait until a matching message arrives. A timeout can also be specified: - -```iex -iex> receive do -...> {:hello, msg} -> msg -...> after -...> 1_000 -> "nothing after 1s" -...> end -"nothing after 1s" -``` - -A timeout of 0 can be given when you already expect the message to be in the mailbox. - -Let's put it all together and send messages between processes: - -```iex -iex> parent = self() -#PID<0.41.0> -iex> spawn fn -> send(parent, {:hello, self()}) end -#PID<0.48.0> -iex> receive do -...> {:hello, pid} -> "Got hello from #{inspect pid}" -...> end -"Got hello from #PID<0.48.0>" -``` - -While in the shell, you may find the helper `flush/0` quite useful. It flushes and prints all the messages in the mailbox. - -```iex -iex> send self(), :hello -:hello -iex> flush() -:hello -:ok -``` - -## Links - -The most common form of spawning in Elixir is actually via `spawn_link/1`. Before we show an example with `spawn_link/1`, let's try to see what happens when a process fails: - -```iex -iex> spawn fn -> raise "oops" end -#PID<0.58.0> - -[error] Process #PID<0.58.00> raised an exception -** (RuntimeError) oops - :erlang.apply/2 -``` - -It merely logged an error but the spawning process is still running. That's because processes are isolated. If we want the failure in one process to propagate to another one, we should link them. This can be done with `spawn_link/1`: - -```iex -iex> spawn_link fn -> raise "oops" end -#PID<0.41.0> - -** (EXIT from #PID<0.41.0>) an exception was raised: - ** (RuntimeError) oops - :erlang.apply/2 -``` - -When a failure happens in the shell, the shell automatically traps the failure and shows it nicely formatted. In order to understand what would really happen in our code, let's use `spawn_link/1` inside a file and run it: - -```iex -# spawn.exs -spawn_link fn -> raise "oops" end - -receive do - :hello -> "let's wait until the process fails" -end -``` - -``` -$ elixir spawn.exs - -** (EXIT from #PID<0.47.0>) an exception was raised: - ** (RuntimeError) oops - spawn.exs:1: anonymous fn/0 in :elixir_compiler_0.__FILE__/1 -``` - -This time the process failed and brought the parent process down as they are linked. Linking can also be done manually by calling `Process.link/1`. We recommend that you take a look at [the `Process` module](https://hexdocs.pm/elixir/Process.html) for other functionality provided by processes. - -Processes and links play an important role when building fault-tolerant systems. In Elixir applications, we often link our processes to supervisors which will detect when a process dies and start a new process in its place. This is only possible because processes are isolated and don't share anything by default. And since processes are isolated, there is no way a failure in a process will crash or corrupt the state of another. - -While other languages would require us to catch/handle exceptions, in Elixir we are actually fine with letting processes fail because we expect supervisors to properly restart our systems. "Failing fast" is a common philosophy when writing Elixir software! - -`spawn/1` and `spawn_link/1` are the basic primitives for creating processes in Elixir. Although we have used them exclusively so far, most of the time we are going to use abstractions that build on top of them. Let's see the most common one, called tasks. - -## Tasks - -Tasks build on top of the spawn functions to provide better error reports and introspection: - -```iex -iex(1)> Task.start fn -> raise "oops" end -{:ok, #PID<0.55.0>} - -15:22:33.046 [error] Task #PID<0.55.0> started from #PID<0.53.0> terminating -** (RuntimeError) oops - (elixir) lib/task/supervised.ex:74: Task.Supervised.do_apply/2 - (stdlib) proc_lib.erl:239: :proc_lib.init_p_do_apply/3 -Function: #Function<20.90072148/0 in :erl_eval.expr/5> - Args: [] -``` - -Instead of `spawn/1` and `spawn_link/1`, we use `Task.start/1` and `Task.start_link/1` to return `{:ok, pid}` rather than just the PID. This is what enables Tasks to be used in supervision trees. Furthermore, `Task` provides convenience functions, like `Task.async/1` and `Task.await/1`, and functionality to ease distribution. - -We will explore those functionalities in the ***Mix and OTP guide***, for now it is enough to remember to use Task to get better error reports. - -## State - -We haven't talked about state so far in this guide. If you are building an application that requires state, for example, to keep your application configuration, or you need to parse a file and keep it in memory, where would you store it? - -Processes are the most common answer to this question. We can write processes that loop infinitely, maintain state, and send and receive messages. As an example, let's write a module that starts new processes that work as a key-value store in a file named `kv.exs`: - -```elixir -defmodule KV do - def start_link do - Task.start_link(fn -> loop(%{}) end) - end - - defp loop(map) do - receive do - {:get, key, caller} -> - send caller, Map.get(map, key) - loop(map) - {:put, key, value} -> - loop(Map.put(map, key, value)) - end - end -end -``` - -Note that the `start_link` function starts a new process that runs the `loop/1` function, starting with an empty map. The `loop/1` function then waits for messages and performs the appropriate action for each message. In the case of a `:get` message, it sends a message back to the caller and calls `loop/1` again, to wait for a new message. While the `:put` message actually invokes `loop/1` with a new version of the map, with the given `key` and `value` stored. - -Let's give it a try by running `iex kv.exs`: - -```iex -iex> {:ok, pid} = KV.start_link -#PID<0.62.0> -iex> send pid, {:get, :hello, self()} -{:get, :hello, #PID<0.41.0>} -iex> flush -nil -:ok -``` - -At first, the process map has no keys, so sending a `:get` message and then flushing the current process inbox returns `nil`. Let's send a `:put` message and try it again: - -```iex -iex> send pid, {:put, :hello, :world} -{:put, :hello, :world} -iex> send pid, {:get, :hello, self()} -{:get, :hello, #PID<0.41.0>} -iex> flush -:world -:ok -``` - -Notice how the process is keeping a state and we can get and update this state by sending the process messages. In fact, any process that knows the `pid` above will be able to send it messages and manipulate the state. - -It is also possible to register the `pid`, giving it a name, and allowing everyone that knows the name to send it messages: - -```iex -iex> Process.register(pid, :kv) -true -iex> send :kv, {:get, :hello, self()} -{:get, :hello, #PID<0.41.0>} -iex> flush -:world -:ok -``` - -Using processes to maintain state and name registration are very common patterns in Elixir applications. However, most of the time, we won't implement those patterns manually as above, but by using one of the many abstractions that ship with Elixir. For example, Elixir provides [agents](https://hexdocs.pm/elixir/Agent.html), which are simple abstractions around state: - -```iex -iex> {:ok, pid} = Agent.start_link(fn -> %{} end) -{:ok, #PID<0.72.0>} -iex> Agent.update(pid, fn map -> Map.put(map, :hello, :world) end) -:ok -iex> Agent.get(pid, fn map -> Map.get(map, :hello) end) -:world -``` - -A `:name` option could also be given to `Agent.start_link/2` and it would be automatically registered. Besides agents, Elixir provides an API for building generic servers (called GenServer), tasks and more, all powered by processes underneath. Those, along with supervision trees, will be explored with more detail in the ***Mix and OTP guide*** which will build a complete Elixir application from start to finish. - -For now, let's move on and explore the world of I/O in Elixir. diff --git a/getting-started/protocols.markdown b/getting-started/protocols.markdown index 88a9214f4..a44082f4e 100644 --- a/getting-started/protocols.markdown +++ b/getting-started/protocols.markdown @@ -1,235 +1,5 @@ --- -layout: getting-started -title: Protocols +layout: redirect +sitemap: false +redirect_to: protocols --- - -# {{ page.title }} - -{% include toc.html %} - -Protocols are a mechanism to achieve polymorphism in Elixir. Dispatching on a protocol is available to any data type as long as it implements the protocol. Let's see an example. - -In Elixir, we have two verbs for checking how many items there are in a data structure: `length` and `size`. `length` means the information must be computed. For example, `length(list)` needs to traverse the whole list to calculate its length. On the other hand, `tuple_size(tuple)` and `byte_size(binary)` do not depend on the tuple and binary size as the size information is precomputed in the data structure. - -Even if we have type-specific functions for getting the size built into Elixir (such as `tuple_size/1`), we could implement a generic `Size` protocol that all data structures for which size is precomputed would implement. - -The protocol definition would look like this: - -```elixir -defprotocol Size do - @doc "Calculates the size (and not the length!) of a data structure" - def size(data) -end -``` - -The `Size` protocol expects a function called `size` that receives one argument (the data structure we want to know the size of) to be implemented. We can now implement this protocol for the data structures that would have a compliant implementation: - -```elixir -defimpl Size, for: Binary do - def size(binary), do: byte_size(binary) -end - -defimpl Size, for: Map do - def size(map), do: map_size(map) -end - -defimpl Size, for: Tuple do - def size(tuple), do: tuple_size(tuple) -end -``` - -We didn't implement the `Size` protocol for lists as there is no "size" information precomputed for lists, and the length of a list has to be computed (with `length/1`). - -Now with the protocol defined and implementations in hand, we can start using it: - -```iex -iex> Size.size("foo") -3 -iex> Size.size({:ok, "hello"}) -2 -iex> Size.size(%{label: "some label"}) -1 -``` - -Passing a data type that doesn't implement the protocol raises an error: - -```iex -iex> Size.size([1, 2, 3]) -** (Protocol.UndefinedError) protocol Size not implemented for [1, 2, 3] -``` - -It's possible to implement protocols for all Elixir data types: - -* `Atom` -* `BitString` -* `Float` -* `Function` -* `Integer` -* `List` -* `Map` -* `PID` -* `Port` -* `Reference` -* `Tuple` - - -## Protocols and structs - -The power of Elixir's extensibility comes when protocols and structs are used together. - -In the [previous chapter](/getting-started/structs.html), we have learned that although structs are maps, they do not share protocol implementations with maps. For example, [`MapSet`](https://hexdocs.pm/elixir/MapSet.html)s (sets based on maps) are implemented as structs. Let's try to use the `Size` protocol with a `MapSet`: - -```iex -iex> Size.size(%{}) -0 -iex> set = %MapSet{} = MapSet.new -iex> Size.size(set) -** (Protocol.UndefinedError) protocol Size not implemented for #MapSet<[]> -``` - -Instead of sharing protocol implementation with maps, structs require their own protocol implementation. Since a `MapSet` has its size precomputed and accessible through `MapSet.size/1`, we can define a `Size` implementation for it: - -```elixir -defimpl Size, for: MapSet do - def size(set), do: MapSet.size(set) -end -``` - -If desired, you could come up with your own semantics for the size of your struct. Not only that, you could use structs to build more robust data types, like queues, and implement all relevant protocols, such as `Enumerable` and possibly `Size`, for this data type. - -```elixir -defmodule User do - defstruct [:name, :age] -end - -defimpl Size, for: User do - def size(_user), do: 2 -end -``` - -## Implementing `Any` - -Manually implementing protocols for all types can quickly become repetitive and tedious. In such cases, Elixir provides two options: we can explicitly derive the protocol implementation for our types or automatically implement the protocol for all types. In both cases, we need to implement the protocol for `Any`. - -### Deriving - -Elixir allows us to derive a protocol implementation based on the `Any` implementation. Let's first implement `Any` as follows: - -```elixir -defimpl Size, for: Any do - def size(_), do: 0 -end -``` - -The implementation above is arguably not a reasonable one. For example, it makes no sense to say that the size of a `PID` or an `Integer` is `0`. - -However, should we be fine with the implementation for `Any`, in order to use such implementation we would need to tell our struct to explicitly derive the `Size` protocol: - -```elixir -defprotocol OtherUser do - @derive [Size] - defstruct [:name, :age] -end -``` - -When deriving, Elixir will implement the `Size` protocol for `OtherUser` based on the implementation provided for `Any`. - -### Fallback to `Any` - -Another alternative to `@derive` is to explicitly tell the protocol to fallback to `Any` when an implementation cannot be found. This can be achieved by setting `@fallback_to_any` to `true` in the protocol definition: - -```elixir -defprotocol Size do - @fallback_to_any true - def size(data) -end -``` - -As we said in the previous section, the implementation of `Size` for `Any` is not one that can apply to any data type. That's one of the reasons why `@fallback_to_any` is an opt-in behaviour. For the majority of protocols, raising an error when a protocol is not implemented is the proper behaviour. That said, assuming we have implemented `Any` as in the previous section: - -```elixir -defimpl Size, for: Any do - def size(_), do: 0 -end -``` - -Now all data types (including structs) that have not implemented the `Size` protocol will be considered to have a size of `0`. - -Which technique is best between deriving and falling back to any depends on the use case but, given Elixir developers prefer explicit over implicit, you may see many libraries pushing towards the `@derive` approach. - -## Built-in protocols - -Elixir ships with some built-in protocols. In previous chapters, we have discussed the `Enum` module which provides many functions that work with any data structure that implements the `Enumerable` protocol: - -```iex -iex> Enum.map [1, 2, 3], fn(x) -> x * 2 end -[2, 4, 6] -iex> Enum.reduce 1..3, 0, fn(x, acc) -> x + acc end -6 -``` -Another useful example is the `String.Chars` protocol, which specifies how to convert a data structure with characters to a string. It's exposed via the `to_string` function: - -```iex -iex> to_string :hello -"hello" -``` - -Notice that string interpolation in Elixir calls the `to_string` function: - -```iex -iex> "age: #{25}" -"age: 25" -``` - -The snippet above only works because numbers implement the `String.Chars` protocol. Passing a tuple, for example, will lead to an error: - -```iex -iex> tuple = {1, 2, 3} -{1, 2, 3} -iex> "tuple: #{tuple}" -** (Protocol.UndefinedError) protocol String.Chars not implemented for {1, 2, 3} -``` - -When there is a need to "print" a more complex data structure, one can use the `inspect` function, based on the `Inspect` protocol: - -```iex -iex> "tuple: #{inspect tuple}" -"tuple: {1, 2, 3}" -``` - -The `Inspect` protocol is the protocol used to transform any data structure into a readable textual representation. This is what tools like IEx use to print results: - -```iex -iex> {1, 2, 3} -{1, 2, 3} -iex> %User{} -%User{name: "john", age: 27} -``` - -Keep in mind that, by convention, whenever the inspected value starts with `#`, it is representing a data structure in non-valid Elixir syntax. This means the inspect protocol is not reversible as information may be lost along the way: - -```iex -iex> inspect &(&1+2) -"#Function<6.71889879/1 in :erl_eval.expr/5>" -``` - -There are other protocols in Elixir but this covers the most common ones. - -## Protocol consolidation - -When working with Elixir projects, using the Mix build tool, you may see output as follows: - -``` -Consolidated String.Chars -Consolidated Collectable -Consolidated List.Chars -Consolidated IEx.Info -Consolidated Enumerable -Consolidated Inspect -``` - -Those are all protocols that ship with Elixir and they are being consolidated. Because a protocol can dispatch to any data type, the protocol must check on every call if an implementation for the given type exists. This may be expensive. - -However, after our project is compiled using a tool like Mix, we know all modules that have been defined, including protocols and their implementations. This way, the protocol can be consolidated into a very simple and fast dispatch module. - -From Elixir v1.2, protocol consolidation happens automatically for all projects. We will build our own project in the ***Mix and OTP guide***. diff --git a/getting-started/recursion.markdown b/getting-started/recursion.markdown index 6d2fbee66..ed7c1c2c9 100644 --- a/getting-started/recursion.markdown +++ b/getting-started/recursion.markdown @@ -1,130 +1,5 @@ --- -layout: getting-started -title: Recursion +layout: redirect +sitemap: false +redirect_to: recursion --- - -# {{ page.title }} - -{% include toc.html %} - -## Loops through recursion - -Due to immutability, loops in Elixir (as in any functional programming language) are written differently from imperative languages. For example, in an imperative language like C, one would write: - -```c -for(i = 0; i < sizeof(array); i++) { - array[i] = array[i] * 2; -} -``` - -In the example above, we are mutating both the array and the variable `i`. Mutating is not possible in Elixir. Instead, functional languages rely on recursion: a function is called recursively until a condition is reached that stops the recursive action from continuing. No data is mutated in this process. Consider the example below that prints a string an arbitrary number of times: - -```elixir -defmodule Recursion do - def print_multiple_times(msg, n) when n <= 1 do - IO.puts msg - end - - def print_multiple_times(msg, n) do - IO.puts msg - print_multiple_times(msg, n - 1) - end -end - -Recursion.print_multiple_times("Hello!", 3) -# Hello! -# Hello! -# Hello! -``` - -Similar to `case`, a function may have many clauses. A particular clause is executed when the arguments passed to the function match the clause's argument patterns and its guard evaluates to `true`. - -When `print_multiple_times/2` is initially called in the example above, the argument `n` is equal to `3`. - -The first clause has a guard which says "use this definition if and only if `n` is less than or equal to `1`". Since this is not the case, Elixir proceeds to the next clause's definition. - -The second definition matches the pattern and has no guard so it will be executed. It first prints our `msg` and then calls itself passing `n - 1` (`2`) as the second argument. - -Our `msg` is printed and `print_multiple_times/2` is called again, this time with the second argument set to `1`. -Because `n` is now set to `1`, the guard in our first definition of `print_multiple_times/2` evaluates to true, and we execute this particular definition. The `msg` is printed, and there is nothing left to execute. - -We defined `print_multiple_times/2` so that, no matter what number is passed as the second argument, it either triggers our first definition (known as a _base case_) or it triggers our second definition, which will ensure that we get exactly one step closer to our base case. - -## Reduce and map algorithms - -Let's now see how we can use the power of recursion to sum a list of numbers: - -```elixir -defmodule Math do - def sum_list([head | tail], accumulator) do - sum_list(tail, head + accumulator) - end - - def sum_list([], accumulator) do - accumulator - end -end - -IO.puts Math.sum_list([1, 2, 3], 0) #=> 6 -``` - -We invoke `sum_list` with the list `[1, 2, 3]` and the initial value `0` as arguments. We will try each clause until we find one that matches according to the pattern matching rules. In this case, the list `[1, 2, 3]` matches against `[head | tail]` which binds `head` to `1` and `tail` to `[2, 3]`; `accumulator` is set to `0`. - -Then, we add the head of the list to the accumulator `head + accumulator` and call `sum_list` again, recursively, passing the tail of the list as its first argument. The tail will once again match `[head | tail]` until the list is empty, as seen below: - -```elixir -sum_list [1, 2, 3], 0 -sum_list [2, 3], 1 -sum_list [3], 3 -sum_list [], 6 -``` - -When the list is empty, it will match the final clause which returns the final result of `6`. - -The process of taking a list and _reducing_ it down to one value is known as a _reduce algorithm_ and is central to functional programming. - -What if we instead want to double all of the values in our list? - -```elixir -defmodule Math do - def double_each([head | tail]) do - [head * 2 | double_each(tail)] - end - - def double_each([]) do - [] - end -end -``` - -```bash -iex math.exs -``` - -```iex -iex> Math.double_each([1, 2, 3]) #=> [2, 4, 6] -``` - -Here we have used recursion to traverse a list, doubling each element and returning a new list. The process of taking a list and _mapping_ over it is known as a _map algorithm_. - -Recursion and [tail call](https://en.wikipedia.org/wiki/Tail_call) optimization are an important part of Elixir and are commonly used to create loops. However, when programming in Elixir you will rarely use recursion as above to manipulate lists. - -The [`Enum` module](https://hexdocs.pm/elixir/Enum.html), which we're going to see in the next chapter, already provides many conveniences for working with lists. For instance, the examples above could be written as: - -```iex -iex> Enum.reduce([1, 2, 3], 0, fn(x, acc) -> x + acc end) -6 -iex> Enum.map([1, 2, 3], fn(x) -> x * 2 end) -[2, 4, 6] -``` - -Or, using the capture syntax: - -```iex -iex> Enum.reduce([1, 2, 3], 0, &+/2) -6 -iex> Enum.map([1, 2, 3], &(&1 * 2)) -[2, 4, 6] -``` - -Let's take a deeper look at `Enumerable`s and, while we're at it, their lazy counterpart, `Stream`s. diff --git a/getting-started/sigils.markdown b/getting-started/sigils.markdown index fbb637891..0f929e472 100644 --- a/getting-started/sigils.markdown +++ b/getting-started/sigils.markdown @@ -1,193 +1,5 @@ --- -layout: getting-started -title: Sigils +layout: redirect +sitemap: false +redirect_to: sigils --- - -# {{ page.title }} - -{% include toc.html %} - -We have already learned that Elixir provides double-quoted strings and single-quoted char lists. However, this only covers the surface of structures that have textual representation in the language. Atoms, for example, are mostly created via the `:atom` representation. - -One of Elixir's goals is extensibility: developers should be able to extend the language to fit any particular domain. Computer science has become such a wide field that it is impossible for a language to tackle many fields as part of its core. Rather, our best bet is to make the language extensible, so developers, companies and communities can extend the language to their relevant domains. - -In this chapter, we are going to explore sigils, which are one of the mechanisms provided by the language for working with textual representations. Sigils start with the tilde (`~`) character which is followed by a letter (which identifies the sigil) and then a delimiter; optionally, modifiers can be added after the final delimiter. - -## Regular expressions - -The most common sigil in Elixir is `~r`, which is used to create [regular expressions](https://en.wikipedia.org/wiki/Regular_Expressions): - -```iex -# A regular expression that matches strings which contain "foo" or "bar": -iex> regex = ~r/foo|bar/ -~r/foo|bar/ -iex> "foo" =~ regex -true -iex> "bat" =~ regex -false -``` - -Elixir provides Perl-compatible regular expressions (regexes), as implemented by the [PCRE](http://www.pcre.org/) library. Regexes also support modifiers. For example, the `i` modifier makes a regular expression case insensitive: - -```iex -iex> "HELLO" =~ ~r/hello/ -false -iex> "HELLO" =~ ~r/hello/i -true -``` - -Check out the [`Regex` module](https://hexdocs.pm/elixir/Regex.html) for more information on other modifiers and the supported operations with regular expressions. - -So far, all examples have used `/` to delimit a regular expression. However sigils support 8 different delimiters: - -``` -~r/hello/ -~r|hello| -~r"hello" -~r'hello' -~r(hello) -~r[hello] -~r{hello} -~r -``` - -The reason behind supporting different delimiters is that different delimiters can be more suited for different sigils. For example, using parentheses for regular expressions may be a confusing choice as they can get mixed with the parentheses inside the regex. However, parentheses can be handy for other sigils, as we will see in the next section. - -## Strings, char lists and words sigils - -Besides regular expressions, Elixir ships with three other sigils. - -### Strings - -The `~s` sigil is used to generate strings, like double quotes are. The `~s` sigil is useful, for example, when a string contains both double and single quotes: - -```iex -iex> ~s(this is a string with "double" quotes, not 'single' ones) -"this is a string with \"double\" quotes, not 'single' ones" -``` - -### Char lists - -The `~c` sigil is used to generate char lists: - -```iex -iex> ~c(this is a char list containing 'single quotes') -'this is a char list containing \'single quotes\'' -``` - -### Word lists - -The `~w` sigil is used to generate lists of words (*words* are just regular strings). Inside the `~w` sigil, words are separated by whitespace. - -```iex -iex> ~w(foo bar bat) -["foo", "bar", "bat"] -``` - -The `~w` sigil also accepts the `c`, `s` and `a` modifiers (for char lists, strings and atoms, respectively), which specify the data type of the elements of the resulting list: - -```iex -iex> ~w(foo bar bat)a -[:foo, :bar, :bat] -``` - -## Interpolation and escaping in sigils - -Besides lowercase sigils, Elixir supports uppercase sigils to deal with escaping characters and interpolation. While both `~s` and `~S` will return strings, the former allows escape codes and interpolation while the latter does not: - -```iex -iex> ~s(String with escape codes \x26 #{"inter" <> "polation"}) -"String with escape codes & interpolation" -iex> ~S(String without escape codes \x26 without #{interpolation}) -"String without escape codes \\x26 without \#{interpolation}" -``` - -The following escape codes can be used in strings and char lists: - -* `\"` – double quote -* `\'` – single quote -* `\\` – single backslash -* `\a` – bell/alert -* `\b` – backspace -* `\d` - delete -* `\e` - escape -* `\f` - form feed -* `\n` – newline -* `\r` – carriage return -* `\s` – space -* `\t` – tab -* `\v` – vertical tab -* `\0` - null byte -* `\xDD` - represents a single byte in hexadecimal (such as `\x13`) -* `\uDDDD` and `\u{D...}` - represents a Unicode codepoint in hexadecimal (such as `\u{1F600}`) - -Sigils also support heredocs, that is, triple double- or single-quotes as separators: - -```iex -iex> ~s""" -...> this is -...> a heredoc string -...> """ -``` - -The most common use case for heredoc sigils is when writing documentation. For example, writing escape characters in documentation would soon become error prone because of the need to double-escape some characters: - -```elixir -@doc """ -Converts double-quotes to single-quotes. - -## Examples - - iex> convert("\\\"foo\\\"") - "'foo'" - -""" -def convert(...) -``` - -By using `~S`, this problem can be avoided altogether: - -```elixir -@doc ~S""" -Converts double-quotes to single-quotes. - -## Examples - - iex> convert("\"foo\"") - "'foo'" - -""" -def convert(...) -``` - -## Custom sigils - -As hinted at the beginning of this chapter, sigils in Elixir are extensible. In fact, using the sigil `~r/foo/i` is equivalent to calling `sigil_r` with a binary and a char list as argument: - -```iex -iex> sigil_r(<<"foo">>, 'i') -~r"foo"i -``` - -We can access the documentation for the `~r` sigil via `sigil_r`: - -```iex -iex> h sigil_r -... -``` - -We can also provide our own sigils by implementing functions that follow the `sigil_{identifier}` pattern. For example, let's implement the `~i` sigil that returns an integer (with the optional `n` modifier to make it negative): - -```iex -iex> defmodule MySigils do -...> def sigil_i(string, []), do: String.to_integer(string) -...> def sigil_i(string, [?n]), do: -String.to_integer(string) -...> end -iex> import MySigils -iex> ~i(13) -13 -iex> ~i(42)n --42 -``` - -Sigils can also be used to do compile-time work with the help of macros. For example, regular expressions in Elixir are compiled into an efficient representation during compilation of the source code, therefore skipping this step at runtime. If you're interested in the subject, we recommend you learn more about macros and check out how sigils are implemented in the `Kernel` module (where the `sigil_*` functions are defined). diff --git a/getting-started/structs.markdown b/getting-started/structs.markdown index 1b0bcddfe..2a92104e6 100644 --- a/getting-started/structs.markdown +++ b/getting-started/structs.markdown @@ -1,139 +1,5 @@ --- -layout: getting-started -title: Structs -redirect_from: /getting-started/struct.html +layout: redirect +sitemap: false +redirect_to: structs --- - -# {{ page.title }} - -{% include toc.html %} - -In [chapter 7](/getting-started/keywords-and-maps.html) we learned about maps: - -```iex -iex> map = %{a: 1, b: 2} -%{a: 1, b: 2} -iex> map[:a] -1 -iex> %{map | a: 3} -%{a: 3, b: 2} -``` - -Structs are extensions built on top of maps that provide compile-time checks and default values. - -## Defining structs - -To define a struct, the `defstruct` construct is used: - -```iex -iex> defmodule User do -...> defstruct name: "John", age: 27 -...> end -``` - -The keyword list used with `defstruct` defines what fields the struct will have along with their default values. - -Structs take the name of the module they're defined in. In the example above, we defined a struct named `User`. - -We can now create `User` structs by using a syntax similar to the one used to create maps: - -```iex -iex> %User{} -%User{age: 27, name: "John"} -iex> %User{name: "Meg"} -%User{age: 27, name: "Meg"} -``` - -Structs provide *compile-time* guarantees that only the fields (and *all* of them) defined through `defstruct` will be allowed to exist in a struct: - -```iex -iex> %User{oops: :field} -** (CompileError) iex:3: unknown key :oops for struct User -``` - -## Accessing and updating structs - -When we discussed maps, we showed how we can access and update the fields of a map. The same techniques (and the same syntax) apply to structs as well: - -```iex -iex> john = %User{} -%User{age: 27, name: "John"} -iex> john.name -"John" -iex> meg = %{john | name: "Meg"} -%User{age: 27, name: "Meg"} -iex> %{meg | oops: :field} -** (KeyError) key :oops not found in: %User{age: 27, name: "Meg"} -``` - -When using the update syntax (`|`), the VM is aware that no new keys will be added to the struct, allowing the maps underneath to share their structure in memory. In the example above, both `john` and `meg` share the same key structure in memory. - -Structs can also be used in pattern matching, both for matching on the value of specific keys as well as for ensuring that the matching value is a struct of the same type as the matched value. - -```iex -iex> %User{name: name} = john -%User{age: 27, name: "John"} -iex> name -"John" -iex> %User{} = %{} -** (MatchError) no match of right hand side value: %{} -``` - -## Structs are bare maps underneath - -In the example above, pattern matching works because underneath structs are bare maps with a fixed set of fields. As maps, structs store a "special" field named `__struct__` that holds the name of the struct: - -```iex -iex> is_map(john) -true -iex> john.__struct__ -User -``` - -Notice that we referred to structs as **bare** maps because none of the protocols implemented for maps are available for structs. For example, you can neither enumerate nor access a struct: - -```iex -iex> john = %User{} -%User{age: 27, name: "John"} -iex> john[:name] -** (UndefinedFunctionError) undefined function: User.fetch/2 -iex> Enum.each john, fn({field, value}) -> IO.puts(value) end -** (Protocol.UndefinedError) protocol Enumerable not implemented for %User{age: 27, name: "John"} -``` - -However, since structs are just maps, they work with the functions from the `Map` module: - -```iex -iex> kurt = Map.put(%User{}, :name, "Kurt") -%User{age: 27, name: "Kurt"} -iex> Map.merge(kurt, %User{name: "Takashi"}) -%User{age: 27, name: "Takashi"} -iex> Map.keys(john) -[:__struct__, :age, :name] -``` - -Structs alongside protocols provide one of the most important features for Elixir developers: data polymorphism. That's what we will explore in the next chapter. - -## Default values and required keys - -If you don't specify a default key value when defining a struct, `nil` will be assumed: - -```iex -iex> defmodule Product do -...> defstruct [:name] -...> end -iex> %Product{} -%Product{name: nil} -``` - -You can also enforce that certain keys have to be specified when creating the struct: - -```iex -iex> defmodule Car do -...> @enforce_keys [:make] -...> defstruct [:model, :make] -...> end -iex> %Car{} -** (ArgumentError) the following keys must also be given when building struct Car: [:make] - expanding struct: Car.__struct__/1 -``` diff --git a/getting-started/try-catch-and-rescue.markdown b/getting-started/try-catch-and-rescue.markdown index 2fb0fcbbb..158dc94ce 100644 --- a/getting-started/try-catch-and-rescue.markdown +++ b/getting-started/try-catch-and-rescue.markdown @@ -1,225 +1,5 @@ --- -layout: getting-started -title: try, catch and rescue +layout: redirect +sitemap: false +redirect_to: try-catch-and-rescue --- - -# {{ page.title }} - -{% include toc.html %} - -Elixir has three error mechanisms: errors, throws and exits. In this chapter we will explore each of them and include remarks about when each should be used. - -## Errors - -Errors (or *exceptions*) are used when exceptional things happen in the code. A sample error can be retrieved by trying to add a number into an atom: - -```iex -iex> :foo + 1 -** (ArithmeticError) bad argument in arithmetic expression - :erlang.+(:foo, 1) -``` - -A runtime error can be raised any time by using `raise/1`: - -```iex -iex> raise "oops" -** (RuntimeError) oops -``` - -Other errors can be raised with `raise/2` passing the error name and a list of keyword arguments: - -```iex -iex> raise ArgumentError, message: "invalid argument foo" -** (ArgumentError) invalid argument foo -``` - -You can also define your own errors by creating a module and using the `defexception` construct inside it; this way, you'll create an error with the same name as the module it's defined in. The most common case is to define a custom exception with a message field: - -```iex -iex> defmodule MyError do -iex> defexception message: "default message" -iex> end -iex> raise MyError -** (MyError) default message -iex> raise MyError, message: "custom message" -** (MyError) custom message -``` - -Errors can be **rescued** using the `try/rescue` construct: - -```iex -iex> try do -...> raise "oops" -...> rescue -...> e in RuntimeError -> e -...> end -%RuntimeError{message: "oops"} -``` - -The example above rescues the runtime error and returns the error itself which is then printed in the `iex` session. - -If you don't have any use for the error, you don't have to provide it: - -```iex -iex> try do -...> raise "oops" -...> rescue -...> RuntimeError -> "Error!" -...> end -"Error!" -``` - -In practice, however, Elixir developers rarely use the `try/rescue` construct. For example, many languages would force you to rescue an error when a file cannot be opened successfully. Elixir instead provides a `File.read/1` function which returns a tuple containing information about whether the file was opened successfully: - -```iex -iex> File.read "hello" -{:error, :enoent} -iex> File.write "hello", "world" -:ok -iex> File.read "hello" -{:ok, "world"} -``` - -There is no `try/rescue` here. In case you want to handle multiple outcomes of opening a file, you can use pattern matching within the `case` construct: - -```iex -iex> case File.read "hello" do -...> {:ok, body} -> IO.puts "Success: #{body}" -...> {:error, reason} -> IO.puts "Error: #{reason}" -...> end -``` - -At the end of the day, it's up to your application to decide if an error while opening a file is exceptional or not. That's why Elixir doesn't impose exceptions on `File.read/1` and many other functions. Instead, it leaves it up to the developer to choose the best way to proceed. - -For the cases where you do expect a file to exist (and the lack of that file is truly an *error*) you may use `File.read!/1`: - -```iex -iex> File.read! "unknown" -** (File.Error) could not read file unknown: no such file or directory - (elixir) lib/file.ex:305: File.read!/1 -``` - -Many functions in the standard library follow the pattern of having a counterpart that raises an exception instead of returning tuples to match against. The convention is to create a function (`foo`) which returns `{:ok, result}` or `{:error, reason}` tuples and another function (`foo!`, same name but with a trailing `!`) that takes the same arguments as `foo` but which raises an exception if there's an error. `foo!` should return the result (not wrapped in a tuple) if everything goes fine. The [`File` module](https://hexdocs.pm/elixir/File.html) is a good example of this convention. - -In Elixir, we avoid using `try/rescue` because **we don't use errors for control flow**. We take errors literally: they are reserved for unexpected and/or exceptional situations. In case you actually need flow control constructs, *throws* should be used. That's what we are going to see next. - -## Throws - -In Elixir, a value can be thrown and later be caught. `throw` and `catch` are reserved for situations where it is not possible to retrieve a value unless by using `throw` and `catch`. - -Those situations are quite uncommon in practice except when interfacing with libraries that do not provide a proper API. For example, let's imagine the `Enum` module did not provide any API for finding a value and that we needed to find the first multiple of 13 in a list of numbers: - -```iex -iex> try do -...> Enum.each -50..50, fn(x) -> -...> if rem(x, 13) == 0, do: throw(x) -...> end -...> "Got nothing" -...> catch -...> x -> "Got #{x}" -...> end -"Got -39" -``` - -Since `Enum` *does* provide a proper API, in practice `Enum.find/2` is the way to go: - -```iex -iex> Enum.find -50..50, &(rem(&1, 13) == 0) --39 -``` - -## Exits - -All Elixir code runs inside processes that communicate with each other. When a process dies of "natural causes" (e.g., unhandled exceptions), it sends an `exit` signal. A process can also die by explicitly sending an exit signal: - -```iex -iex> spawn_link fn -> exit(1) end -#PID<0.56.0> -** (EXIT from #PID<0.56.0>) 1 -``` - -In the example above, the linked process died by sending an `exit` signal with value of 1. The Elixir shell automatically handles those messages and prints them to the terminal. - -`exit` can also be "caught" using `try/catch`: - -```iex -iex> try do -...> exit "I am exiting" -...> catch -...> :exit, _ -> "not really" -...> end -"not really" -``` - -Using `try/catch` is already uncommon and using it to catch exits is even more rare. - -`exit` signals are an important part of the fault tolerant system provided by the Erlang VM. Processes usually run under supervision trees which are themselves processes that listen to `exit` signals from the supervised processes. Once an exit signal is received, the supervision strategy kicks in and the supervised process is restarted. - -It is exactly this supervision system that makes constructs like `try/catch` and `try/rescue` so uncommon in Elixir. Instead of rescuing an error, we'd rather "fail fast" since the supervision tree will guarantee our application will go back to a known initial state after the error. - -## After - -Sometimes it's necessary to ensure that a resource is cleaned up after some action that could potentially raise an error. The `try/after` construct allows you to do that. For example, we can open a file and use an `after` clause to close it--even if something goes wrong: - -```iex -iex> {:ok, file} = File.open "sample", [:utf8, :write] -iex> try do -...> IO.write file, "olá" -...> raise "oops, something went wrong" -...> after -...> File.close(file) -...> end -** (RuntimeError) oops, something went wrong -``` - -The `after` clause will be executed regardless of whether or not the tried block succeeds. Note, however, that if a linked process exits, -this process will exit and the `after` clause will not get run. Thus `after` provides only a soft guarantee. Luckily, files in Elixir are also linked to the current processes and therefore they will always get closed if the current process crashes, independent of the -`after` clause. You will find the same to be true for other resources like ETS tables, sockets, ports and more. - -Sometimes you may want to wrap the entire body of a function in a `try` construct, often to guarantee some code will be executed afterwards. In such cases, Elixir allows you to omit the `try` line: - -```iex -iex> defmodule RunAfter do -...> def without_even_trying do -...> raise "oops" -...> after -...> IO.puts "cleaning up!" -...> end -...> end -iex> RunAfter.without_even_trying -cleaning up! -** (RuntimeError) oops -``` - -Elixir will automatically wrap the function body in a `try` whenever one of `after`, `rescue` or `catch` is specified. - -## Variables scope - -It is important to bear in mind that variables defined inside `try/catch/rescue/after` blocks do not leak to the outer context. This is because the `try` block may fail and as such the variables may never be bound in the first place. In other words, this code is invalid: - -```iex -iex> try do -...> raise "fail" -...> what_happened = :did_not_raise -...> rescue -...> _ -> what_happened = :rescued -...> end -iex> what_happened -** (RuntimeError) undefined function: what_happened/0 -``` - -Instead, you can store the value of the `try` expression: - -```iex -iex> what_happened = -...> try do -...> raise "fail" -...> :did_not_raise -...> rescue -...> _ -> :rescued -...> end -iex> what_happened -:rescued -``` - -This finishes our introduction to `try`, `catch` and `rescue`. You will find they are used less frequently in Elixir than in other languages, although they may be handy in some situations where a library or some particular code is not playing "by the rules". diff --git a/getting-started/typespecs-and-behaviours.markdown b/getting-started/typespecs-and-behaviours.markdown index 101576bb7..af6ce2592 100644 --- a/getting-started/typespecs-and-behaviours.markdown +++ b/getting-started/typespecs-and-behaviours.markdown @@ -1,137 +1,5 @@ --- -layout: getting-started -title: Typespecs and behaviours +layout: redirect +sitemap: false +redirect_to: typespecs --- - -# {{ page.title }} - -{% include toc.html %} - -## Types and specs - -Elixir is a dynamically typed language, so all types in Elixir are inferred by the runtime. Nonetheless, Elixir comes with **typespecs**, which are a notation used for: - -1. declaring custom data types; -2. declaring typed function signatures (specifications). - -### Function specifications - -By default, Elixir provides some basic types, such as `integer` or `pid`, as well as more complex types: for example, the `round/1` function, which rounds a float to its nearest integer, takes a `number` as an argument (an `integer` or a `float`) and returns an `integer`. As you can see [in its documentation](https://hexdocs.pm/elixir/Kernel.html#round/1), `round/1`'s typed signature is written as: - -``` -round(number) :: integer -``` - -`::` means that the function on the left side *returns* a value whose type is what's on the right side. Function specs are written with the `@spec` directive, placed right before the function definition. The `round/1` function could be written as: - -```elixir -@spec round(number) :: integer -def round(number), do: # implementation... -``` - -Elixir supports compound types as well. For example, a list of integers has type `[integer]`. You can see all the built-in types provided by Elixir [in the typespecs docs](https://hexdocs.pm/elixir/typespecs.html). - -### Defining custom types - -While Elixir provides a lot of useful built-in types, it's convenient to define custom types when appropriate. This can be done when defining modules through the `@type` directive. - -Say we have a `LousyCalculator` module, which performs the usual arithmetic operations (sum, product and so on) but, instead of returning numbers, it returns tuples with the result of an operation as the first element and a random remark as the second element. - -```elixir -defmodule LousyCalculator do - @spec add(number, number) :: {number, String.t} - def add(x, y), do: {x + y, "You need a calculator to do that?!"} - - @spec multiply(number, number) :: {number, String.t} - def multiply(x, y), do: {x * y, "Jeez, come on!"} -end -``` - -As you can see in the example, tuples are a compound type and each tuple is identified by the types inside it. To understand why `String.t` is not written as `string`, have another look at the [notes in the typespecs docs](https://hexdocs.pm/elixir/typespecs.html#notes). - -Defining function specs this way works, but it quickly becomes annoying since we're repeating the type `{number, String.t}` over and over. We can use the `@type` directive in order to declare our own custom type. - -```elixir -defmodule LousyCalculator do - @typedoc """ - Just a number followed by a string. - """ - @type number_with_remark :: {number, String.t} - - @spec add(number, number) :: number_with_remark - def add(x, y), do: {x + y, "You need a calculator to do that?"} - - @spec multiply(number, number) :: number_with_remark - def multiply(x, y), do: {x * y, "It is like addition on steroids."} -end -``` - -The `@typedoc` directive, similarly to the `@doc` and `@moduledoc` directives, is used to document custom types. - -Custom types defined through `@type` are exported and available outside the module they're defined in: - -```elixir -defmodule QuietCalculator do - @spec add(number, number) :: number - def add(x, y), do: make_quiet(LousyCalculator.add(x, y)) - - @spec make_quiet(LousyCalculator.number_with_remark) :: number - defp make_quiet({num, _remark}), do: num -end -``` - -If you want to keep a custom type private, you can use the `@typep` directive instead of `@type`. - -### Static code analysis - -Typespecs are not only useful to developers and as additional documentation. The Erlang tool [Dialyzer](http://www.erlang.org/doc/man/dialyzer.html), for example, uses typespecs in order to perform static analysis of code. That's why, in the `QuietCalculator` example, we wrote a spec for the `make_quiet/1` function even if it was defined as a private function. - -## Behaviours - -Many modules share the same public API. Take a look at [Plug](https://github.com/elixir-lang/plug), which, as its description states, is a **specification** for composable modules in web applications. Each *plug* is a module which **has to** implement at least two public functions: `init/1` and `call/2`. - -Behaviours provide a way to: - -* define a set of functions that have to be implemented by a module; -* ensure that a module implements all the functions in that set. - -If you have to, you can think of behaviours like interfaces in object oriented languages like Java: a set of function signatures that a module has to implement. - -### Defining behaviours - -Say we want to implement a bunch of parsers, each parsing structured data: for example, a JSON parser and a YAML parser. Each of these two parsers will *behave* the same way: both will provide a `parse/1` function and an `extensions/0` function. The `parse/1` function will return an Elixir representation of the structured data, while the `extensions/0` function will return a list of file extensions that can be used for each type of data (e.g., `.json` for JSON files). - -We can create a `Parser` behaviour: - -```elixir -defmodule Parser do - @callback parse(String.t) :: any - @callback extensions() :: [String.t] -end -``` - -Modules adopting the `Parser` behaviour will have to implement all the functions defined with the `@callback` directive. As you can see, `@callback` expects a function name but also a function specification like the ones used with the `@spec` directive we saw above. - -### Adopting behaviours - -Adopting a behaviour is straightforward: - -```elixir -defmodule JSONParser do - @behaviour Parser - - def parse(str), do: # ... parse JSON - def extensions, do: ["json"] -end -``` - -```elixir -defmodule YAMLParser do - @behaviour Parser - - def parse(str), do: # ... parse YAML - def extensions, do: ["yml"] -end -``` - -If a module adopting a given behaviour doesn't implement one of the callbacks required by that behaviour, a compile-time warning will be generated. diff --git a/getting-started/where-to-go-next.markdown b/getting-started/where-to-go-next.markdown index baaf86564..c50365d75 100644 --- a/getting-started/where-to-go-next.markdown +++ b/getting-started/where-to-go-next.markdown @@ -1,44 +1,5 @@ --- -layout: getting-started -title: Where to go next +layout: redirect +sitemap: false +redirect_to: introduction --- - -# {{ page.title }} - -{% include toc.html %} - -Eager to learn more? Keep reading! - -## Build your first Elixir project - -In order to get your first project started, Elixir ships with a build tool called Mix. You can get your new project started by running: - -```bash -$ mix new path/to/new/project -``` - -We have written a guide that covers how to build an Elixir application, with its own supervision tree, configuration, tests and more. The application works as a distributed key-value store where we organize key-value pairs into buckets and distribute those buckets across multiple nodes: - -* [Mix and OTP](/getting-started/mix-otp/introduction-to-mix.html) - -## Meta-programming - -Elixir is an extensible and very customizable programming language thanks to its meta-programming support. Most meta-programming in Elixir is done through macros, which are very useful in several situations, especially for writing DSLs. We have written a short guide that explains the basic mechanisms behind macros, shows how to write macros, and how to use macros to create DSLs: - -* [Meta-programming in Elixir](/getting-started/meta/quote-and-unquote.html) - -## Community and other resources - -We have a [Learning](/learning.html) section that suggests books, screencasts and other resources for learning Elixir and exploring the ecosystem. There are also plenty of Elixir resources out there, like conference talks, open source projects, and other learning material produced by the community. - -Don't forget that you can also check the [source code of Elixir itself](https://github.com/elixir-lang/elixir), which is mostly written in Elixir (mainly the `lib` directory), or [explore Elixir's documentation](/docs.html). - -## A byte of Erlang - -Elixir runs on the Erlang Virtual Machine and, sooner or later, an Elixir developer will want to interface with existing Erlang libraries. Here's a list of online resources that cover Erlang's fundamentals and its more advanced features: - -* This [Erlang Syntax: A Crash Course](/crash-course.html) provides a concise intro to Erlang's syntax. Each code snippet is accompanied by equivalent code in Elixir. This is an opportunity for you to not only get some exposure to Erlang's syntax but also review some of the things you have learned in this guide. - -* Erlang's official website has a short [tutorial](http://www.erlang.org/course/concurrent_programming.html) with pictures that briefly describe Erlang's primitives for concurrent programming. - -* [Learn You Some Erlang for Great Good!](http://learnyousomeerlang.com/) is an excellent introduction to Erlang, its design principles, standard library, best practices and much more. Once you have read through the crash course mentioned above, you'll be able to safely skip the first couple of chapters in the book that mostly deal with the syntax. When you reach [The Hitchhiker's Guide to Concurrency](http://learnyousomeerlang.com/the-hitchhikers-guide-to-concurrency) chapter, that's where the real fun starts. diff --git a/images/cases/bg/change-graph.png b/images/cases/bg/change-graph.png new file mode 100644 index 000000000..41e70fc34 Binary files /dev/null and b/images/cases/bg/change-graph.png differ diff --git a/images/cases/bg/change.png b/images/cases/bg/change.png new file mode 100644 index 000000000..7bf0d37a0 Binary files /dev/null and b/images/cases/bg/change.png differ diff --git a/images/cases/bg/cyanview-1.jpg b/images/cases/bg/cyanview-1.jpg new file mode 100644 index 000000000..2e0b9ffcf Binary files /dev/null and b/images/cases/bg/cyanview-1.jpg differ diff --git a/images/cases/bg/cyanview-2.jpg b/images/cases/bg/cyanview-2.jpg new file mode 100644 index 000000000..a6e7796e6 Binary files /dev/null and b/images/cases/bg/cyanview-2.jpg differ diff --git a/images/cases/bg/cyanview-3.jpg b/images/cases/bg/cyanview-3.jpg new file mode 100644 index 000000000..fbe3515f3 Binary files /dev/null and b/images/cases/bg/cyanview-3.jpg differ diff --git a/images/cases/bg/cyanview-4.jpg b/images/cases/bg/cyanview-4.jpg new file mode 100644 index 000000000..c14baed64 Binary files /dev/null and b/images/cases/bg/cyanview-4.jpg differ diff --git a/images/cases/bg/discord.jpg b/images/cases/bg/discord.jpg new file mode 100755 index 000000000..06e482dd7 Binary files /dev/null and b/images/cases/bg/discord.jpg differ diff --git a/images/cases/bg/duffel.png b/images/cases/bg/duffel.png new file mode 100644 index 000000000..d06a6e9a6 Binary files /dev/null and b/images/cases/bg/duffel.png differ diff --git a/images/cases/bg/farmbot.jpg b/images/cases/bg/farmbot.jpg new file mode 100644 index 000000000..da5dda3e3 Binary files /dev/null and b/images/cases/bg/farmbot.jpg differ diff --git a/images/cases/bg/heroku.png b/images/cases/bg/heroku.png new file mode 100644 index 000000000..04454907f Binary files /dev/null and b/images/cases/bg/heroku.png differ diff --git a/images/cases/bg/met-france.png b/images/cases/bg/met-france.png new file mode 100644 index 000000000..c79da2fb1 Binary files /dev/null and b/images/cases/bg/met-france.png differ diff --git a/images/cases/bg/mozilla-hubs.jpg b/images/cases/bg/mozilla-hubs.jpg new file mode 100644 index 000000000..5b78639d9 Binary files /dev/null and b/images/cases/bg/mozilla-hubs.jpg differ diff --git a/images/cases/bg/pepsico.jpg b/images/cases/bg/pepsico.jpg new file mode 100644 index 000000000..e9f6be8de Binary files /dev/null and b/images/cases/bg/pepsico.jpg differ diff --git a/images/cases/bg/remote.png b/images/cases/bg/remote.png new file mode 100644 index 000000000..a4285ad4e Binary files /dev/null and b/images/cases/bg/remote.png differ diff --git a/images/cases/bg/slab.png b/images/cases/bg/slab.png new file mode 100644 index 000000000..6f5a0e3dc Binary files /dev/null and b/images/cases/bg/slab.png differ diff --git a/images/cases/bg/sparkmeter-new-architecture.png b/images/cases/bg/sparkmeter-new-architecture.png new file mode 100644 index 000000000..9ed82cabe Binary files /dev/null and b/images/cases/bg/sparkmeter-new-architecture.png differ diff --git a/images/cases/bg/sparkmeter-old-architecture.png b/images/cases/bg/sparkmeter-old-architecture.png new file mode 100644 index 000000000..a5acca6ba Binary files /dev/null and b/images/cases/bg/sparkmeter-old-architecture.png differ diff --git a/images/cases/bg/v7.png b/images/cases/bg/v7.png new file mode 100644 index 000000000..59391bf33 Binary files /dev/null and b/images/cases/bg/v7.png differ diff --git a/images/cases/bg/x-plane.jpg b/images/cases/bg/x-plane.jpg new file mode 100644 index 000000000..10da67180 Binary files /dev/null and b/images/cases/bg/x-plane.jpg differ diff --git a/images/cases/logos/change.png b/images/cases/logos/change.png new file mode 100644 index 000000000..9d002c59a Binary files /dev/null and b/images/cases/logos/change.png differ diff --git a/images/cases/logos/community.png b/images/cases/logos/community.png new file mode 100644 index 000000000..11e6cff7b Binary files /dev/null and b/images/cases/logos/community.png differ diff --git a/images/cases/logos/cyanview.png b/images/cases/logos/cyanview.png new file mode 100644 index 000000000..4fac3bf69 Binary files /dev/null and b/images/cases/logos/cyanview.png differ diff --git a/images/cases/logos/default-image.png b/images/cases/logos/default-image.png new file mode 100644 index 000000000..edcfde32a Binary files /dev/null and b/images/cases/logos/default-image.png differ diff --git a/images/cases/logos/discord.png b/images/cases/logos/discord.png new file mode 100644 index 000000000..aa1b5e2e0 Binary files /dev/null and b/images/cases/logos/discord.png differ diff --git a/images/cases/logos/duffel.png b/images/cases/logos/duffel.png new file mode 100644 index 000000000..a2c197a8b Binary files /dev/null and b/images/cases/logos/duffel.png differ diff --git a/images/cases/logos/farmbot.png b/images/cases/logos/farmbot.png new file mode 100644 index 000000000..d8d5a914b Binary files /dev/null and b/images/cases/logos/farmbot.png differ diff --git a/images/cases/logos/heroku.png b/images/cases/logos/heroku.png new file mode 100644 index 000000000..78f52eda4 Binary files /dev/null and b/images/cases/logos/heroku.png differ diff --git a/images/cases/logos/met-france.svg b/images/cases/logos/met-france.svg new file mode 100644 index 000000000..09ff2ae0a --- /dev/null +++ b/images/cases/logos/met-france.svg @@ -0,0 +1,3 @@ + + +Accueil - Ministère de la Transition écologiqueMinistère de la Transition écologique diff --git a/images/cases/logos/mozilla-hubs.png b/images/cases/logos/mozilla-hubs.png new file mode 100644 index 000000000..f1b7779c3 Binary files /dev/null and b/images/cases/logos/mozilla-hubs.png differ diff --git a/images/cases/logos/pepsico.png b/images/cases/logos/pepsico.png new file mode 100644 index 000000000..308de0c1f Binary files /dev/null and b/images/cases/logos/pepsico.png differ diff --git a/images/cases/logos/remote.png b/images/cases/logos/remote.png new file mode 100644 index 000000000..6e8db442f Binary files /dev/null and b/images/cases/logos/remote.png differ diff --git a/images/cases/logos/slab.png b/images/cases/logos/slab.png new file mode 100644 index 000000000..9dcf90f89 Binary files /dev/null and b/images/cases/logos/slab.png differ diff --git a/images/cases/logos/sparkmeter.png b/images/cases/logos/sparkmeter.png new file mode 100644 index 000000000..500414dcb Binary files /dev/null and b/images/cases/logos/sparkmeter.png differ diff --git a/images/cases/logos/v7.png b/images/cases/logos/v7.png new file mode 100644 index 000000000..9acd9dfb7 Binary files /dev/null and b/images/cases/logos/v7.png differ diff --git a/images/cases/logos/veeps.svg b/images/cases/logos/veeps.svg new file mode 100755 index 000000000..8e9ce4fa5 --- /dev/null +++ b/images/cases/logos/veeps.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/images/cases/logos/x-plane.png b/images/cases/logos/x-plane.png new file mode 100644 index 000000000..40da79cc0 Binary files /dev/null and b/images/cases/logos/x-plane.png differ diff --git a/images/contents/debugger-elixir.gif b/images/contents/debugger-elixir.gif new file mode 100644 index 000000000..0d99d2a71 Binary files /dev/null and b/images/contents/debugger-elixir.gif differ diff --git a/images/contents/deps-tree-phoenix.svg b/images/contents/deps-tree-phoenix.svg index 1ebd94a19..23f1146fb 100644 --- a/images/contents/deps-tree-phoenix.svg +++ b/images/contents/deps-tree-phoenix.svg @@ -1,90 +1 @@ - - - - - - -dependency tree - - -phoenix - -phoenix - - -poison - -poison - - -phoenix->poison - - -~> 1.5 or ~> 2.0 - - -phoenix_pubsub - -phoenix_pubsub - - -phoenix->phoenix_pubsub - - -~> 1.0.0-rc - - -cowboy - -cowboy - - -phoenix->cowboy - - -~> 1.0 - - -plug - -plug - - -phoenix->plug - - -~> 1.1 - - -cowlib - -cowlib - - -cowboy->cowlib - - -~> 1.0.0 - - -ranch - -ranch - - -cowboy->ranch - - -~> 1.0 - - -plug->cowboy - - -~> 1.0 - - - +phoenixpoison~> 1.5 or ~> 2.0phoenix_pubsub~> 1.0.0-rccowboy~> 1.0plug~> 1.1cowlib~> 1.0.0ranch~> 1.0~> 1.0 \ No newline at end of file diff --git a/images/contents/exunit-ansi.png b/images/contents/exunit-ansi.png index dd7851e0f..eef145e9e 100644 Binary files a/images/contents/exunit-ansi.png and b/images/contents/exunit-ansi.png differ diff --git a/images/contents/exunit-bare-assertion-diff.png b/images/contents/exunit-bare-assertion-diff.png new file mode 100644 index 000000000..57a7bf7e7 Binary files /dev/null and b/images/contents/exunit-bare-assertion-diff.png differ diff --git a/images/contents/exunit-diff.png b/images/contents/exunit-diff.png index fb94c55b2..4bb568d91 100644 Binary files a/images/contents/exunit-diff.png and b/images/contents/exunit-diff.png differ diff --git a/images/contents/fast-fail.png b/images/contents/fast-fail.png index b7ed7483b..4f2938511 100644 Binary files a/images/contents/fast-fail.png and b/images/contents/fast-fail.png differ diff --git a/images/contents/gem.jpeg b/images/contents/gem.jpeg new file mode 100644 index 000000000..220a094c6 Binary files /dev/null and b/images/contents/gem.jpeg differ diff --git a/images/contents/hash-dict-fetch.png b/images/contents/hash-dict-fetch.png index 38217577b..6af8ad47d 100644 Binary files a/images/contents/hash-dict-fetch.png and b/images/contents/hash-dict-fetch.png differ diff --git a/images/contents/hash-dict-update.png b/images/contents/hash-dict-update.png index 8e7e728d4..d5a00d498 100644 Binary files a/images/contents/hash-dict-update.png and b/images/contents/hash-dict-update.png differ diff --git a/images/contents/home-code.png b/images/contents/home-code.png deleted file mode 100644 index 2e7238c0c..000000000 Binary files a/images/contents/home-code.png and /dev/null differ diff --git a/images/contents/iex-auto-reload.mp4 b/images/contents/iex-auto-reload.mp4 new file mode 100644 index 000000000..d21ad3f8b Binary files /dev/null and b/images/contents/iex-auto-reload.mp4 differ diff --git a/images/contents/iex-coloring.png b/images/contents/iex-coloring.png new file mode 100644 index 000000000..7e3e4a67a Binary files /dev/null and b/images/contents/iex-coloring.png differ diff --git a/images/contents/iex-metadata.png b/images/contents/iex-metadata.png new file mode 100644 index 000000000..709922955 Binary files /dev/null and b/images/contents/iex-metadata.png differ diff --git a/images/contents/iex-pry.png b/images/contents/iex-pry.png index 642d4667a..6c7ae953b 100644 Binary files a/images/contents/iex-pry.png and b/images/contents/iex-pry.png differ diff --git a/images/contents/kv-observer.png b/images/contents/kv-observer.png index 337a37983..7527d7e5c 100644 Binary files a/images/contents/kv-observer.png and b/images/contents/kv-observer.png differ diff --git a/images/contents/livebook-boot-1.15.png b/images/contents/livebook-boot-1.15.png new file mode 100644 index 000000000..f603151fc Binary files /dev/null and b/images/contents/livebook-boot-1.15.png differ diff --git a/images/contents/livebook-compile-1.15.png b/images/contents/livebook-compile-1.15.png new file mode 100644 index 000000000..58f41504f Binary files /dev/null and b/images/contents/livebook-compile-1.15.png differ diff --git a/images/contents/string-help.png b/images/contents/string-help.png index a079ccfe6..3831a5362 100644 Binary files a/images/contents/string-help.png and b/images/contents/string-help.png differ diff --git a/images/contents/type-warning-case.png b/images/contents/type-warning-case.png new file mode 100644 index 000000000..118dad1f1 Binary files /dev/null and b/images/contents/type-warning-case.png differ diff --git a/images/contents/type-warning-function-clause.png b/images/contents/type-warning-function-clause.png new file mode 100644 index 000000000..4c2fa76dd Binary files /dev/null and b/images/contents/type-warning-function-clause.png differ diff --git a/images/contents/type-warning-in-editor.png b/images/contents/type-warning-in-editor.png new file mode 100644 index 000000000..38293a0c8 Binary files /dev/null and b/images/contents/type-warning-in-editor.png differ diff --git a/images/contents/type-warning-on-date-comparison.png b/images/contents/type-warning-on-date-comparison.png new file mode 100644 index 000000000..349372cd6 Binary files /dev/null and b/images/contents/type-warning-on-date-comparison.png differ diff --git a/images/contents/type-warning-on-struct-field.png b/images/contents/type-warning-on-struct-field.png new file mode 100644 index 000000000..1d003bf3c Binary files /dev/null and b/images/contents/type-warning-on-struct-field.png differ diff --git a/images/contents/type-warning-private.png b/images/contents/type-warning-private.png new file mode 100644 index 000000000..da05ab1cb Binary files /dev/null and b/images/contents/type-warning-private.png differ diff --git a/images/learning/adopting-elixir.jpg b/images/learning/adopting-elixir.jpg new file mode 100644 index 000000000..77ec6a2ea Binary files /dev/null and b/images/learning/adopting-elixir.jpg differ diff --git a/images/learning/alchemist-camp.png b/images/learning/alchemist-camp.png new file mode 100644 index 000000000..f6704071c Binary files /dev/null and b/images/learning/alchemist-camp.png differ diff --git a/images/learning/concurrent-data-processing.jpg b/images/learning/concurrent-data-processing.jpg new file mode 100644 index 000000000..5c9bd0a95 Binary files /dev/null and b/images/learning/concurrent-data-processing.jpg differ diff --git a/images/learning/designing-elixir-systems.jpg b/images/learning/designing-elixir-systems.jpg new file mode 100644 index 000000000..68413bd94 Binary files /dev/null and b/images/learning/designing-elixir-systems.jpg differ diff --git a/images/learning/educative-io-metaprogramming.png b/images/learning/educative-io-metaprogramming.png new file mode 100644 index 000000000..bf143b471 Binary files /dev/null and b/images/learning/educative-io-metaprogramming.png differ diff --git a/images/learning/elixir-cookbook.jpg b/images/learning/elixir-cookbook.jpg index b702431a2..ecd8fe352 100644 Binary files a/images/learning/elixir-cookbook.jpg and b/images/learning/elixir-cookbook.jpg differ diff --git a/images/learning/elixir-in-action.jpg b/images/learning/elixir-in-action.jpg index b19112a13..932bce213 100644 Binary files a/images/learning/elixir-in-action.jpg and b/images/learning/elixir-in-action.jpg differ diff --git a/images/learning/elixir-koans-logo.png b/images/learning/elixir-koans-logo.png new file mode 100644 index 000000000..fe9de9f24 Binary files /dev/null and b/images/learning/elixir-koans-logo.png differ diff --git a/images/learning/elixir-school.jpg b/images/learning/elixir-school.jpg index ede4f27c1..61dc24175 100644 Binary files a/images/learning/elixir-school.jpg and b/images/learning/elixir-school.jpg differ diff --git a/images/learning/elixir-sips.png b/images/learning/elixir-sips.png index cf84e6d1e..3c6aea8f0 100644 Binary files a/images/learning/elixir-sips.png and b/images/learning/elixir-sips.png differ diff --git a/images/learning/elixir-streams-logo.png b/images/learning/elixir-streams-logo.png new file mode 100644 index 000000000..48e4434d3 Binary files /dev/null and b/images/learning/elixir-streams-logo.png differ diff --git a/images/learning/elixircasts.png b/images/learning/elixircasts.png new file mode 100644 index 000000000..b0052df5b Binary files /dev/null and b/images/learning/elixircasts.png differ diff --git a/images/learning/erlang-in-anger.png b/images/learning/erlang-in-anger.png index 17aaded61..9ba80a420 100644 Binary files a/images/learning/erlang-in-anger.png and b/images/learning/erlang-in-anger.png differ diff --git a/images/learning/etude-for-elixir.jpg b/images/learning/etude-for-elixir.jpg index 3d335d525..549fafad4 100644 Binary files a/images/learning/etude-for-elixir.jpg and b/images/learning/etude-for-elixir.jpg differ diff --git a/images/learning/flashcards_army.jpg b/images/learning/flashcards_army.jpg new file mode 100644 index 000000000..f40790ea6 Binary files /dev/null and b/images/learning/flashcards_army.jpg differ diff --git a/images/learning/groxio-elixir.png b/images/learning/groxio-elixir.png new file mode 100644 index 000000000..08949d5d3 Binary files /dev/null and b/images/learning/groxio-elixir.png differ diff --git a/images/learning/groxio-otp.png b/images/learning/groxio-otp.png new file mode 100644 index 000000000..bf919c08a Binary files /dev/null and b/images/learning/groxio-otp.png differ diff --git a/images/learning/introducing-elixir.jpg b/images/learning/introducing-elixir.jpg index 735c03326..751ef4219 100644 Binary files a/images/learning/introducing-elixir.jpg and b/images/learning/introducing-elixir.jpg differ diff --git a/images/learning/joy-of-elixir.jpg b/images/learning/joy-of-elixir.jpg new file mode 100644 index 000000000..c197a1060 Binary files /dev/null and b/images/learning/joy-of-elixir.jpg differ diff --git a/images/learning/learn-elixir-dev.png b/images/learning/learn-elixir-dev.png new file mode 100644 index 000000000..63793ba8e Binary files /dev/null and b/images/learning/learn-elixir-dev.png differ diff --git a/images/learning/learn-elixir-tv.png b/images/learning/learn-elixir-tv.png index 257de906a..3a134669b 100644 Binary files a/images/learning/learn-elixir-tv.png and b/images/learning/learn-elixir-tv.png differ diff --git a/images/learning/learn-functional-programming-with-elixir.jpg b/images/learning/learn-functional-programming-with-elixir.jpg new file mode 100644 index 000000000..0251fd91b Binary files /dev/null and b/images/learning/learn-functional-programming-with-elixir.jpg differ diff --git a/images/learning/metaprogramming-elixir.jpg b/images/learning/metaprogramming-elixir.jpg index 75fa4190f..44610ed70 100644 Binary files a/images/learning/metaprogramming-elixir.jpg and b/images/learning/metaprogramming-elixir.jpg differ diff --git a/images/learning/mini-docu.png b/images/learning/mini-docu.png new file mode 100644 index 000000000..3a18117f8 Binary files /dev/null and b/images/learning/mini-docu.png differ diff --git a/images/learning/pragmaticstudio-elixir.png b/images/learning/pragmaticstudio-elixir.png new file mode 100644 index 000000000..0c8ab4100 Binary files /dev/null and b/images/learning/pragmaticstudio-elixir.png differ diff --git a/images/learning/programming-elixir-1-3.jpg b/images/learning/programming-elixir-1-3.jpg deleted file mode 100644 index 8fe089986..000000000 Binary files a/images/learning/programming-elixir-1-3.jpg and /dev/null differ diff --git a/images/learning/programming-elixir-1-6.jpg b/images/learning/programming-elixir-1-6.jpg new file mode 100644 index 000000000..b61274161 Binary files /dev/null and b/images/learning/programming-elixir-1-6.jpg differ diff --git a/images/learning/running-in-production-logo.jpg b/images/learning/running-in-production-logo.jpg new file mode 100644 index 000000000..93ba4578a Binary files /dev/null and b/images/learning/running-in-production-logo.jpg differ diff --git a/images/learning/take-off-with-elixir.jpg b/images/learning/take-off-with-elixir.jpg new file mode 100644 index 000000000..c6b4ca144 Binary files /dev/null and b/images/learning/take-off-with-elixir.jpg differ diff --git a/images/learning/techschool.png b/images/learning/techschool.png new file mode 100644 index 000000000..3305082ee Binary files /dev/null and b/images/learning/techschool.png differ diff --git a/images/learning/the-little-elixir-and-otp-guidebook-meap.jpg b/images/learning/the-little-elixir-and-otp-guidebook-meap.jpg index 1bb460400..7ab2cf67d 100644 Binary files a/images/learning/the-little-elixir-and-otp-guidebook-meap.jpg and b/images/learning/the-little-elixir-and-otp-guidebook-meap.jpg differ diff --git a/images/learning/thinking-elixir-pattern-matching-course-tile-190px.png b/images/learning/thinking-elixir-pattern-matching-course-tile-190px.png new file mode 100644 index 000000000..9ac20f687 Binary files /dev/null and b/images/learning/thinking-elixir-pattern-matching-course-tile-190px.png differ diff --git a/images/learning/toy-robot.png b/images/learning/toy-robot.png new file mode 100644 index 000000000..bfbc6a932 Binary files /dev/null and b/images/learning/toy-robot.png differ diff --git a/images/logo/eef.png b/images/logo/eef.png new file mode 100644 index 000000000..0ffba654e Binary files /dev/null and b/images/logo/eef.png differ diff --git a/images/logo/logo-dark.png b/images/logo/logo-dark.png new file mode 100644 index 000000000..8beb02d62 Binary files /dev/null and b/images/logo/logo-dark.png differ diff --git a/images/logo/logo.png b/images/logo/logo.png old mode 100644 new mode 100755 index 003479fd8..9c13fb5a1 Binary files a/images/logo/logo.png and b/images/logo/logo.png differ diff --git a/images/logo/plataformatec.png b/images/logo/plataformatec.png deleted file mode 100644 index 7f6234772..000000000 Binary files a/images/logo/plataformatec.png and /dev/null differ diff --git a/images/social/RSSButton.png b/images/social/RSSButton.png deleted file mode 100644 index d873f9453..000000000 Binary files a/images/social/RSSButton.png and /dev/null differ diff --git a/images/social/elixir-og-card.jpg b/images/social/elixir-og-card.jpg new file mode 100644 index 000000000..64cd0da91 Binary files /dev/null and b/images/social/elixir-og-card.jpg differ diff --git a/images/social/rss-feed-icon.svg b/images/social/rss-feed-icon.svg new file mode 100644 index 000000000..0435819b1 --- /dev/null +++ b/images/social/rss-feed-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/index.html b/index.html index 97f56354f..66925b0e5 100644 --- a/index.html +++ b/index.html @@ -1,18 +1,48 @@ --- section: home layout: default +image: /images/social/elixir-og-card.jpg ---
      -
      - Elixir Sample - +
      -

      Elixir is a dynamic, functional language designed for building scalable and maintainable applications.

      +
      Elixir is a dynamic, functional language for building scalable and maintainable applications.
      + +

      Elixir runs on the Erlang VM, known for creating low-latency, distributed, and fault-tolerant systems. These capabilities and Elixir tooling allow developers to be productive in several domains, such as web development, embedded software, machine learning, data pipelines, and multimedia processing, across a wide range of industries.

      + +

      Here is a peek:

      + +{% highlight elixir %} +iex> "Elixir" |> String.graphemes() |> Enum.frequencies() +%{"E" => 1, "i" => 2, "l" => 1, "r" => 1, "x" => 1} +{% endhighlight %} + +

      Check our Getting Started guide and our Learning page to begin your journey with Elixir. Or keep scrolling for an overview of the platform, language, and tools. +

      +
      -

      Elixir leverages the Erlang VM, known for running low-latency, distributed and fault-tolerant systems, while also being successfully used in web development and the embedded software domain.

      +
      +
      +

      Companies using Elixir in production

      + See more cases → +
      -

      To learn more about Elixir, check our getting started guide. Or keep reading to get an overview of the platform, language and tools.

      +
      @@ -23,26 +53,25 @@

      Platform features

      Scalability

      -

      All Elixir code runs inside lightweight threads of execution (called processes) that are isolated and exchange information via messages:

      {% highlight elixir %} current_process = self() -# Spawns an Elixir process (not an operating system one!) +# Spawn an Elixir process (not an operating system one!) spawn_link(fn -> - send current_process, {:msg, "hello world"} + send(current_process, {:msg, "hello world"}) end) # Block until the message is received receive do - {:msg, contents} -> IO.puts contents + {:msg, contents} -> IO.puts(contents) end {% endhighlight %} -

      Due to their lightweight nature, it is not uncommon to have hundreds of thousands of processes running concurrently in the same machine. Isolation allows processes to be garbage collected independently, reducing system-wide pauses, and using all machine resources as efficiently as possible (vertical scaling).

      +

      Due to their lightweight nature, you can run hundreds of thousands of processes concurrently in the same machine, using all machine resources efficiently (vertical scaling). Processes may also communicate with other processes running on different machines to coordinate work across multiple nodes (horizontal scaling).

      -

      Processes are also able to communicate with other processes running on different machines in the same network. This provides the foundation for distribution, allowing developers to coordinate work across multiple nodes (horizontal scaling).

      +

      Together with projects such as Numerical Elixir, Elixir scales across cores, clusters, and GPUs.

      @@ -50,20 +79,20 @@

      Scalability

      Fault-tolerance

      -

      The unavoidable truth about software running in production is that things will go wrong. Even more when we take network, file systems, and other third-party resources into account.

      +

      The unavoidable truth about software in production is that things will go wrong. Even more when we take network, file systems, and other third-party resources into account.

      -

      To cope with failures, Elixir provides supervisors which describe how to restart parts of your system when things go awry, going back to a known initial state that is guaranteed to work:

      +

      To react to failures, Elixir supervisors describe how to restart parts of your system when things go awry, going back to a known initial state that is guaranteed to work:

      {% highlight elixir %} -import Supervisor.Spec - children = [ - supervisor(TCP.Pool, []), - worker(TCP.Acceptor, [4040]) + TCP.Pool, + {TCP.Acceptor, port: 4040} ] Supervisor.start_link(children, strategy: :one_for_one) {% endhighlight %} + +

      The combination of fault-tolerance and message passing makes Elixir an excellent choice for event-driven systems and robust architectures. Frameworks, such as Nerves, build on this foundation to enable productive development of reliable embedded/IoT systems.

      @@ -75,25 +104,18 @@

      Language features

      Functional programming

      -

      Functional programming promotes a coding style that helps developers write code that is short, fast, and maintainable. For example, pattern matching allows developers to easily destructure data and access its contents:

      - -{% highlight elixir %} -%User{name: name, age: age} = User.get("John Doe") -name #=> "John Doe" -{% endhighlight %} - -

      When mixed with guards, pattern matching allows us to elegantly match and assert specific conditions for some code to execute:

      +

      Functional programming promotes a coding style that helps developers write code that is short, concise, and maintainable. For example, pattern matching allows us to elegantly match and assert specific conditions for some code to execute:

      {% highlight elixir %} -def serve_drinks(%User{age: age}) when age >= 21 do - # Code that serves drinks! +def drive(%User{age: age}) when age >= 16 do + # Code that drives a car end -serve_drinks User.get("John Doe") -#=> Fails if the user is under 21 +drive(User.get("John Doe")) +#=> Fails if the user is under 16 {% endhighlight %} -

      Elixir relies heavily on those features to ensure your software is working under the expected constraints. And when it is not, don’t worry, supervisors have your back!

      +

      Elixir relies on those features to ensure your software is working under the expected constraints. And when it is not, don't worry, supervisors have your back!

      @@ -101,9 +123,9 @@

      Functional programming

      Extensibility and DSLs

      -

      Elixir has been designed to be extensible, letting developers naturally extend the language to particular domains, in order to increase their productivity.

      +

      Elixir has been designed to be extensible, allowing developers to naturally extend the language to particular domains, in order to increase their productivity.

      -

      As an example, let’s write a simple test case using Elixir’s test framework called ExUnit:

      +

      As an example, let's write a simple test case using Elixir's test framework called ExUnit:

      {% highlight elixir %} defmodule MathTest do @@ -115,7 +137,9 @@

      Extensibility and DSLs

      end {% endhighlight %} -

      The async: true option allows tests to run in parallel, using as many CPU cores as possible, while the assert functionality can introspect your code, providing great reports in case of failures. Those features are built using Elixir macros, making it possible to add new constructs as if they were part of the language itself.

      +

      The async: true option allows tests to run in parallel, using as many CPU cores as possible, while the assert functionality can introspect your code, providing great reports in case of failures.

      + +

      Other examples include using Elixir to write SQL queries, compiling a subset of Elixir to the GPU, and more.

      @@ -129,17 +153,16 @@

      A growing ecosystem

      Elixir ships with a great set of tools to ease development. Mix is a build tool that allows you to easily create projects, manage tasks, run tests and more:

      -{% highlight text %} -$ mix new my_app +
      $ mix new my_app
 $ cd my_app
 $ mix test
-.
+.
 
 Finished in 0.04 seconds (0.04s on load, 0.00s on tests)
-1 tests, 0 failures
-{% endhighlight %}
+1 test, 0 failures
+
      -

      Mix is also able to manage dependencies and integrates nicely with the Hex package manager, which provides dependency resolution and the ability to remotely fetch packages.

      +

      Mix also integrates with the Hex package manager for dependency management and hosting documentation for the whole ecosystem.

@@ -147,16 +170,18 @@

A growing ecosystem

Interactive development

-

Tools like IEx (Elixir’s interactive shell) are able to leverage many aspects of the language and platform to provide auto-complete, debugging tools, code reloading, as well as nicely formatted documentation:

+

Tools like IEx (Elixir's interactive shell) leverage the language and platform to provide auto-complete, debugging tools, code reloading, as well as nicely formatted documentation:

{% highlight text %} $ iex Interactive Elixir - press Ctrl+C to exit (type h() ENTER for help) -iex> c "my_file.ex" # Compiles a file -iex> t Enum # Prints types defined in the module Enum -iex> h IEx.pry # Prints the documentation for IEx pry functionality -iex> i "Hello, World" # Prints information about the given data type +iex> h String.trim # Prints the documentation +iex> i "Hello, World" # Prints information about a data type +iex> break! String.trim/1 # Sets a breakpoint +iex> recompile # Recompiles the current project {% endhighlight %} + +

Code notebooks like Livebook allow you to interact with Elixir directly from your browser, including support for plotting, flowcharts, data tables, machine learning, and much more!

@@ -164,14 +189,14 @@

Interactive development

Erlang compatible

-

Elixir runs on the Erlang VM giving developers complete access to Erlang’s ecosystem, used by companies like Heroku, WhatsApp, Klarna, Basho and many more to build distributed, fault-tolerant applications. An Elixir programmer can invoke any Erlang function with no runtime cost:

+

Elixir runs on the Erlang VM giving developers complete access to Erlang's ecosystem, used by companies like WhatsApp, Klarna, and many more to build distributed, fault-tolerant applications. An Elixir programmer can invoke any Erlang function with no runtime cost:

-{% highlight iex %} -iex> :crypto.md5("Using crypto from Erlang OTP") +{% highlight elixir %} +iex> :crypto.hash(:sha256, "Using crypto from Erlang OTP") <<192, 223, 75, 115, ...>> {% endhighlight %} -

To learn more about Elixir, check our getting started guide. We also have online documentation available and a Crash Course for Erlang developers.

+

To learn more about Elixir, check our Getting Started guide.

diff --git a/install.bat b/install.bat new file mode 100755 index 000000000..a6e88101c --- /dev/null +++ b/install.bat @@ -0,0 +1,207 @@ +@echo off +REM See latest version at: +REM https://github.com/elixir-lang/elixir-lang.github.com/blob/main/install.bat + +setlocal EnableDelayedExpansion + +set "otp_version=" +set "elixir_version=" +set "force=false" + +goto :main + +:usage +echo Usage: install.bat elixir@ELIXIR_VERSION otp@OTP_VERSION [options] +echo. +echo ELIXIR_VERSION can be X.Y.Z, latest, or main. +echo OTP_VERSION can be X.Y.Z or latest. +echo. +echo Options: +echo. +echo -f, --force Forces installation even if it was previously installed +echo -h, --help Prints this help +echo. +echo Examples: +echo. +echo install.bat elixir@1.16.3 otp@26.2.5.4 +echo install.bat elixir@latest otp@latest +echo install.bat elixir@main otp@latest +echo. +goto :eof + +:main +for %%i in (%*) do ( + set arg=%%i + + if "!arg:~0,7!" == "elixir@" ( + set "elixir_version=!arg:~7!" + ) else if "!arg:~0,4!" == "otp@" ( + set "otp_version=!arg:~4!" + ) else if "!arg!" == "-f" ( + set "force=true" + ) else if "!arg!" == "--force" ( + set "force=true" + ) else if "!arg!" == "-h" ( + call :usage + exit /b 0 + ) else if "!arg!" == "--help" ( + call :usage + exit /b 0 + ) else ( + echo error: unknown argument !arg! + exit /b 1 + ) +) + +if "%elixir_version%" == "" ( + call :usage + echo error: missing elixir@VERSION argument + exit /b 1 +) + +if "%otp_version%" == "" ( + call :usage + echo error: missing otp@VERSION argument + exit /b 1 +) + +if "!otp_version!" == "latest" ( + set "url=https://github.com/erlang/otp/releases/latest" + for /f "tokens=2 delims= " %%a in ('curl -fsS --head "!url!" ^| findstr /I "^location:"') do set url=%%a + set "otp_version=!url:*releases/tag/OTP-=!" +) + +if "!elixir_version!" == "latest" ( + set "url=https://github.com/elixir-lang/elixir/releases/latest" + for /f "tokens=2 delims= " %%a in ('curl -fsS --head "!url!" ^| findstr /I "^location:"') do set url=%%a + set "elixir_version=!url:*releases/tag/v=!" +) + +for /f "tokens=1 delims=." %%A in ("!otp_version!") do set "elixir_otp_release=%%A" +for /f "tokens=1,2 delims=." %%A in ("!elixir_version!") do set "elixir_major_minor=%%A.%%B" + +if "%elixir_major_minor%" == "1.14" ( + if %elixir_otp_release% GEQ 25 set "elixir_otp_release=25" +) else if "%elixir_major_minor%" == "1.15" ( + if %elixir_otp_release% GEQ 26 set "elixir_otp_release=26" +) else if "%elixir_major_minor%" == "1.16" ( + if %elixir_otp_release% GEQ 26 set "elixir_otp_release=26" +) else if "%elixir_major_minor%" == "1.17" ( + if %elixir_otp_release% GEQ 27 set "elixir_otp_release=27" +) else if "%elixir_major_minor%" == "1.18" ( + if %elixir_otp_release% GEQ 27 set "elixir_otp_release=27" +) else if "%elixir_major_minor%" == "1.19" ( + if %elixir_otp_release% GEQ 28 set "elixir_otp_release=28" +) else ( + if %elixir_otp_release% GEQ 28 set "elixir_otp_release=28" +) + +set "root_dir=%USERPROFILE%\.elixir-install" +set "tmp_dir=%root_dir%\tmp" +mkdir %tmp_dir% 2>nul +set "otp_dir=%root_dir%\installs\otp\%otp_version%" +set "elixir_dir=%root_dir%\installs\elixir\%elixir_version%-otp-%elixir_otp_release%" + +call :install_otp +if %errorlevel% neq 0 exit /b 1 + +set /p="checking OTP... "Previous Elixir versions are available in our [Releases](https://github.com/elixir-lang/elixir/releases) page. + + - Using [install scripts](#install-scripts) + + - Using [Scoop](https://scoop.sh/): + * Install Erlang: `scoop install erlang` + * Install Elixir: `scoop install elixir` + + - Using [Chocolatey](https://community.chocolatey.org/): + * Install Elixir (installs Erlang as a dependency): `choco install elixir` + +### Raspberry Pi and embedded devices + +To build and package an Elixir application, with the whole operating system, and burn that into a disk or deploy it overwhere, [check out the Nerves project](https://www.nerves-project.org). + +If you want to install Elixir as part of an existing Operating System, please follow the relevant steps above for your Operating System or install from precompiled/source. ### Docker -If you are familiar with Docker you can use the offical Docker image to get started quickly with Elixir. +If you are familiar with Docker you can use the official Docker image to get started quickly with Elixir. * Enter interactive mode * Run: `docker run -it --rm elixir` * Enter bash within container with installed `elixir` * Run: `docker run -it --rm elixir bash` -Those distributions will likely install Erlang automatically for you too. In case they don't, check the [Installing Erlang](/install.html#installing-erlang) section below. +The above will automatically point to the latest Erlang and Elixir available. For production usage, we recommend using [Hex.pm Docker images](https://hub.docker.com/r/hexpm/elixir), which are immutable and point to a specific Erlang and Elixir version. + +## Install scripts + +Elixir and Erlang/OTP can be quickly installed for macOS, Windows, or Ubuntu using an `install.sh`/`install.bat` script: + +If you are using bash (macOS/Ubuntu/Windows), run: + +```sh +curl -fsSO {{ site.url }}/install.sh +sh install.sh elixir@{{ stable.version }} otp@{{ stable.recommended_otp }} +installs_dir=$HOME/.elixir-install/installs +export PATH=$installs_dir/otp/{{ stable.recommended_otp }}/bin:$PATH +export PATH=$installs_dir/elixir/{{ stable.version }}-otp-{{ stable.otp_versions[0] }}/bin:$PATH +iex +``` + +If you are using PowerShell (Windows), run: + +```pwsh +curl.exe -fsSO {{ site.url }}/install.bat +.\install.bat elixir@{{ stable.version }} otp@{{ stable.recommended_otp }} +$installs_dir = "$env:USERPROFILE\.elixir-install\installs" +$env:PATH = "$installs_dir\otp\{{ stable.recommended_otp }}\bin;$env:PATH" +$env:PATH = "$installs_dir\elixir\{{ stable.version }}-otp-{{ stable.otp_versions[0] }}\bin;$env:PATH" +iex.bat +``` + +You may want to [set the $PATH environment variable](#setting-path-environment-variable) for your whole system. Use `install.sh --help` or `install.bat --help` to learn more about available arguments and options. ## Precompiled package -Elixir provides a precompiled package for every release. First [install Erlang](/install.html#installing-erlang) and then download and unzip the [Precompiled.zip file for the latest release](https://github.com/elixir-lang/elixir/releases/). +Elixir provides a precompiled package for every release. First [install Erlang](/install.html#installing-erlang) and then download the appropriate precompiled Elixir below. You can consult your Erlang/OTP version by running `erl -s halt`: + +{% for otp_version in stable.otp_versions %} + * [Elixir {{ stable.version }} on Erlang/OTP {{ otp_version }}](https://github.com/elixir-lang/elixir/releases/download/v{{ stable.version }}/elixir-otp-{{ otp_version }}.zip){% endfor %} + +Once you download the release, unpack it, and you are ready to run the `elixir` and `iex` commands from the `bin` directory. However, we recommend you to [add Elixir's bin path to your PATH environment variable](#setting-path-environment-variable) to ease development. + +### Mirrors and nightly builds -Once the release is unpacked, you are ready to run the `elixir` and `iex` commands from the `bin` directory, but we recommend you to [add Elixir's bin path to your PATH environment variable](#setting-path-environment-variable) to ease development. +The links above point directly to the GitHub release. We also host and mirror precompiled packages and nightly builds globally via `builds.hex.pm` using the following URL scheme: -## Compiling with version managers + https://builds.hex.pm/builds/elixir/${ELIXIR_VERSION}-otp-${OTP_VERSION}.zip -There are many tools that allow developers to install and manage multiple Erlang and Elixir versions. They are useful if you can't install Erlang or Elixir as mentioned above or if your package manager is simply outdated. Here are some of those tools: +For example, to use Elixir v1.13.3 with Erlang/OTP 24.x, use: - * [asdf](https://github.com/asdf-vm/asdf) - install and manage different Elixir and Erlang versions - * [exenv](https://github.com/mururu/exenv) - install and manage different Elixir versions - * [kiex](https://github.com/taylor/kiex) - install and manage different Elixir versions - * [kerl](https://github.com/yrashk/kerl) - install and manage different Erlang versions + https://builds.hex.pm/builds/elixir/v1.13.3-otp-24.zip -If you would prefer to compile from source manually, don't worry, we got your back too! +To use nightly for a given Erlang/OTP version (such as 25), use: -## Compiling from source (Unix and MinGW) + https://builds.hex.pm/builds/elixir/main-otp-25.zip -You can download and compile Elixir in few steps. The first one is to [install Erlang](/install.html#installing-erlang). +For a list of all builds, use: -Next you should download the [latest release](https://github.com/elixir-lang/elixir/releases/), unpack it and then run `make` inside the unpacked directory (note: if you are running on Windows, [read this page on setting up your environment for compiling Elixir](https://github.com/elixir-lang/elixir/wiki/Windows)). + https://builds.hex.pm/builds/elixir/builds.txt + +## Compiling from source + +You can download and compile Elixir in few steps. The first one is to [install Erlang](/install.html#installing-erlang). You will also need [make](https://www.gnu.org/software/make/) available. + +Next you should download source code ([.zip](https://github.com/elixir-lang/elixir/archive/v{{ stable.version }}.zip), [.tar.gz](https://github.com/elixir-lang/elixir/archive/v{{ stable.version }}.tar.gz)) of the [latest release](https://github.com/elixir-lang/elixir/releases/tag/v{{ stable.version }}), unpack it and then run `make` inside the unpacked directory (note: if you are running on Windows, [read this page on setting up your environment for compiling Elixir](https://github.com/elixir-lang/elixir/wiki/Windows)). After compiling, you are ready to run the elixir and `iex` commands from the bin directory. It is recommended that you [add Elixir's bin path to your PATH environment variable](#setting-path-environment-variable) to ease development. -In case you are feeling a bit more adventurous, you can also compile from master: +In case you are feeling a bit more adventurous, you can also compile from main: ```bash $ git clone https://github.com/elixir-lang/elixir.git $ cd elixir -$ make clean test +$ make clean compile ``` -If the tests pass, you are ready to go. Otherwise, feel free to open an issue [in the issues tracker on Github](https://github.com/elixir-lang/elixir). - ## Installing Erlang -The only prerequisite for Elixir is Erlang, version 18.0 or later, which can be easily installed with [Precompiled packages](https://www.erlang-solutions.com/resources/download.html). In case you want to install it directly from source, it can be found on [the Erlang website](http://www.erlang.org/download.html) or by following the excellent tutorial available in the [Riak documentation](https://docs.basho.com/riak/latest/ops/building/installing/erlang/). +The only prerequisite for Elixir is Erlang, version {{ stable.minimum_otp }} or later. When installing Elixir, Erlang is generally installed automatically for you. However, if you want to install Erlang manually, you might check: -For Windows developers, we recommend the precompiled packages. Those on a Unix platform can probably get Erlang installed via one of the many package distribution tools. + * [Source code distribution and Windows installers from Erlang's official website](http://www.erlang.org/downloads.html) + * [Precompiled packages for some Unix-like installations](https://www.erlang-solutions.com/resources/download.html) + * [A general list of installation methods from the Riak documentation](https://docs.riak.com/riak/kv/latest/setup/installing/source/erlang/) -After Erlang is installed, you should be able to open up the command line (or command prompt) and check the Erlang version by typing `erl`. You will see some information as follows: +After Erlang is installed, you should be able to open up the command line (or command prompt) and check the Erlang version by typing `erl -s erlang halt`. You will see some information similar to: - Erlang/OTP 18 (erts-7) [64-bit] [smp:2:2] [async-threads:0] [hipe] [kernel-poll:false] + Erlang/OTP {{ stable.minimum_otp }} [64-bit] [smp:2:2] [...] Notice that depending on how you installed Erlang, Erlang binaries might not be available in your PATH. Be sure to have Erlang binaries in your [PATH](https://en.wikipedia.org/wiki/Environment_variable), otherwise Elixir won't work! @@ -120,10 +210,28 @@ Notice that depending on how you installed Erlang, Erlang binaries might not be It is highly recommended to add Elixir's bin path to your PATH environment variable to ease development. -On **Windows**, there are [instructions for different versions](http://www.computerhope.com/issues/ch000549.htm) explaining the process. +On Windows, there are [instructions for different versions](http://www.computerhope.com/issues/ch000549.htm) explaining the process. -On **Unix systems**, you need to [find your shell profile file](https://unix.stackexchange.com/a/117470/101951), and then add to the end of this file the following line reflecting the path to your Elixir installation: +On Unix systems, you need to [find your shell profile file](https://unix.stackexchange.com/a/117470/101951), and then add to the end of this file the following line reflecting the path to your Elixir installation: ```bash export PATH="$PATH:/path/to/elixir/bin" ``` + +## Asking questions + +After Elixir is up and running, it is common to have questions as you learn and use the language. There are many places where you can ask questions, here are some of them: + + * [#elixir on irc.libera.chat](irc://irc.libera.chat/elixir) + * [Elixir Forum](http://elixirforum.com) + * [Elixir on Slack](https://elixir-slack.community) + * [Elixir on Discord](https://discord.gg/elixir) + * [elixir tag on StackOverflow](https://stackoverflow.com/questions/tagged/elixir) + +When asking questions, remember these two tips: + + * Instead of asking "how to do X in Elixir", ask "how to solve Y in Elixir". In other words, don't ask how to implement a particular solution, instead describe the problem at hand. Stating the problem gives more context and less bias for a correct answer. + + * In case things are not working as expected, please include as much information as you can in your report, for example: your Elixir version, the code snippet and the error message alongside the error stacktrace. + +Enjoy! diff --git a/install.sh b/install.sh new file mode 100755 index 000000000..f1647a15f --- /dev/null +++ b/install.sh @@ -0,0 +1,298 @@ +#!/bin/sh +# See latest version at: +# https://github.com/elixir-lang/elixir-lang.github.com/blob/main/install.sh + +set -eu + +otp_version= +elixir_version= +force=false + +usage() { + cat<&2 + exit 1 + ;; + esac + done + + if [ -z "${elixir_version}" ]; then + usage + echo "error: missing elixir@VERSION argument" + exit 1 + fi + + if [ -z "${otp_version}" ]; then + usage + echo "error: missing otp@VERSION argument" + exit 1 + fi + + root_dir="$HOME/.elixir-install" + tmp_dir="$root_dir/tmp" + mkdir -p "$tmp_dir" + + if [ "${otp_version}" = latest ]; then + url=$(curl -fsS --head https://github.com/erlef/otp_builds/releases/latest | grep -i '^location:' | awk '{print $2}' | tr -d '\r\n') + tag=$(basename "$url") + otp_version="${tag#OTP-}" + fi + + if [ "${elixir_version}" = latest ]; then + url=$(curl -fsS --head https://github.com/elixir-lang/elixir/releases/latest | grep -i '^location:' | awk '{print $2}' | tr -d '\r\n') + tag=$(basename "$url") + elixir_version="${tag#v}" + fi + + case "${otp_version}" in + master|maint*) + branch_version=$(curl -fsS https://raw.githubusercontent.com/erlang/otp/refs/heads/${otp_version}/OTP_VERSION | tr -d '\n') + elixir_otp_release="${branch_version%%.*}" + ;; + *) + elixir_otp_release="${otp_version%%.*}" + ;; + esac + + case "$elixir_version" in + 1.14.*) + [ "${elixir_otp_release}" -ge 25 ] && elixir_otp_release=25 + ;; + 1.15.*|1.16.*) + [ "${elixir_otp_release}" -ge 26 ] && elixir_otp_release=26 + ;; + 1.17.*|1.18.*) + [ "${elixir_otp_release}" -ge 27 ] && elixir_otp_release=27 + ;; + 1.19.*) + [ "${elixir_otp_release}" -ge 28 ] && elixir_otp_release=28 + ;; + *) + [ "${elixir_otp_release}" -ge 28 ] && elixir_otp_release=28 + ;; + esac + + otp_dir="$root_dir/installs/otp/$otp_version" + elixir_dir="${root_dir}/installs/elixir/${elixir_version}-otp-${elixir_otp_release}" + + if unzip_available; then + install_otp & + pid_otp=$! + + install_elixir & + pid_elixir=$! + + wait $pid_otp + wait $pid_elixir + else + # if unzip is missing (e.g. official docker ubuntu image), install otp and elixir + # serially because we unzip elixir using OTP zip:extract/2. + install_otp + install_elixir + fi + + printf "checking OTP... " + export PATH="$otp_dir/bin:$PATH" + erl -noshell -eval 'io:put_chars(erlang:system_info(otp_release) ++ " ok\n"), halt().' + + printf "checking Elixir... " + "$elixir_dir/bin/elixir" -e 'IO.puts(System.version() <> " ok")' + + export PATH="$elixir_dir/bin:$PATH" +cat</dev/null 2>&1 +} + +main "$@" diff --git a/js/index.js b/js/index.js new file mode 100644 index 000000000..cb605e3a8 --- /dev/null +++ b/js/index.js @@ -0,0 +1,163 @@ +// https://github.com/ghiculescu/jekyll-table-of-contents +$.fn.toc = function(options) { + var defaults = { + title: '', + minimumHeaders: 3, + headers: 'h1, h2, h3, h4, h5, h6', + listType: 'ol', // values: [ol|ul] + + linkHeader: true, + linkHere: false, + linkHereText: '', + linkHereTitle: 'Link here', + backToTop: false, + backToTopId: '', + backToTopText: '', + backToTopTitle: 'Back to top', + backToTopDisplay: 'always', // values: [always|highest] + }, + settings = $.extend(defaults, options); + + var headers = $(settings.headers).filter(function() { + // get all headers with an ID + var previousSiblingName = $(this).prev().attr( "name" ); + if (!this.id && previousSiblingName) { + this.id = $(this).attr( "id", previousSiblingName.replace(/\./g, "-") ); + } + return this.id; + }), output = $(this); + + if (!headers.length || headers.length < settings.minimumHeaders || !output.length) { + return; + } + + var get_level = function(ele) { return parseInt(ele.nodeName.replace("H", ""), 10); } + var highest_level = headers.map(function(_, ele) { return get_level(ele); }).get().sort()[0]; + + var level = get_level(headers[0]), + this_level, + html = settings.title + ' <'+settings.listType+' class="jekyll-toc">'; + + var back_to_top = function(id) { + return ''+settings.backToTopText+''; + } + + var link_here = function(id) { + return ''+settings.linkHereText+''; + } + + function fixedEncodeURIComponent (str) { + return encodeURIComponent(str).replace(/[!'()*]/g, function(c) { + return '%' + c.charCodeAt(0).toString(16); + }); + } + + function force_update_hash(hash) { + if ( window.location.hash == hash ) { + window.location.hash = ''; + } + window.location.hash = hash; + + } + + $(headers).each(function(_, header) { + this_level = get_level(header); + var header_id = $(header).attr('id'); + if (this_level === level) // same level as before; same indenting + html += "
  • " + header.innerHTML + ""; + else if (this_level <= level){ // higher level than before; end parent ol + for(i = this_level; i < level; i++) { + html += "
    • " + } + html += "
  • " + header.innerHTML + ""; + } + else if (this_level > level) { // lower level than before; expand the previous to contain a ol + for(i = this_level; i > level; i--) { + html += "<"+settings.listType+">
  • " + } + html += "" + header.innerHTML + ""; + } + level = this_level; // update for the next one + + // add links at the end (so we don't pulute header.innerHTML) + $(header).addClass('jekyll-toc-top-level-header').wrapInner('').append( link_here(header_id) ); + if (settings.backToTop) { + switch(settings.backToTopDisplay){ + case 'highest': + if ( this_level === highest_level ) { + $(header).append( back_to_top(settings.backToTopId) ); + } + break; + case 'always': + default: + $(header).append( back_to_top(settings.backToTopId) ); + } + } + + if (settings.linkHeader) { + $(header).addClass('jekyll-toc-header'); + $(header).children('span.jekyll-toc-wrapper').on( 'click', function( ) { + force_update_hash(fixedEncodeURIComponent(header_id)); + }); + } + }); + + html += ""; + + output.html(html) +}; + +// https://css-tricks.com/snippets/jquery/shuffle-dom-elements/ +$.fn.shuffle = function() { + var allElems = this.get(), + getRandom = function(max) { + return Math.floor(Math.random() * max); + }, + shuffled = $.map(allElems, function(){ + var random = getRandom(allElems.length), + randEl = $(allElems[random]).clone(true)[0]; + allElems.splice(random, 1); + return randEl; + }); + + this.each(function(i){ + $(this).replaceWith($(shuffled[i])); + }); + + return $(shuffled); +}; + +// use plugins +$(document).ready(function() { + $('.toc').toc({ + title: '', + listType: 'ol', + minimumHeaders: 2, + headers: 'h2, h3, h4, h5, h6', + linkHere: true, + linkHereTitle: 'Link here', + backToTop: true, + backToTopId: 'toc', + backToTopTitle: 'Back to Table of Contents', + }); + + $('.jekyll-toc-header a.jekyll-toc-link-here span.jekyll-toc-icon').addClass('icon icon-link'); + $('.jekyll-toc-header a.jekyll-toc-back-to-top span.jekyll-toc-icon').addClass('icon icon-chevron-up'); + + $('#top-banner .close').click(function() { + $(this).parent().slideUp(200); + document.cookie = 'topBannerDisabled=true'; + }) + + $("#shuffled-cases").children().shuffle() +}); + +// Automatically redirect to https +(function (){ + const location = window.location.href; + const localhost = /:\/\/(?:localhost|127.0.0.1|::1)/ + + if (location.startsWith("http://") && !localhost.test(location)) { + window.location.href = location.replace("http://", "https://"); + } +})(); \ No newline at end of file diff --git a/js/toc/README.md b/js/toc/README.md index 0ac0dfe5d..2d9412d0f 100644 --- a/js/toc/README.md +++ b/js/toc/README.md @@ -60,12 +60,12 @@ Will render this table of contents: 3. Page 2 a. Note on Paragraph 2 b. Note on Paragraph 4 - + ### Configuration #### List Type By default the table of contents is rendered as an `
      `, so you can change the number formatting using CSS. -However you can use the `
        ` tag, using the `listType` option: +However, you can use the `
          ` tag, using the `listType` option: ```javascript $('#toc').toc({ listType: 'ul' }); @@ -100,7 +100,7 @@ Otherwise, you can use the stylesheet below to have the icon and the header alig ``` #### Headers Used -By default the table of content is displayed when at least 3 headers are found. +By default the table of content is displayed when at least 3 headers are found. You can customize the minimum number of headers required with this setting: ```javascript diff --git a/js/toc/toc.js b/js/toc/toc.js deleted file mode 100644 index 5f4cf9b6b..000000000 --- a/js/toc/toc.js +++ /dev/null @@ -1,110 +0,0 @@ -// https://github.com/ghiculescu/jekyll-table-of-contents -(function($){ - $.fn.toc = function(options) { - var defaults = { - title: '', - minimumHeaders: 3, - headers: 'h1, h2, h3, h4, h5, h6', - listType: 'ol', // values: [ol|ul] - - linkHeader: true, - linkHere: false, - linkHereText: '', - linkHereTitle: 'Link here', - backToTop: false, - backToTopId: '', - backToTopText: '', - backToTopTitle: 'Back to top', - backToTopDisplay: 'always', // values: [always|highest] - }, - settings = $.extend(defaults, options); - - var headers = $(settings.headers).filter(function() { - // get all headers with an ID - var previousSiblingName = $(this).prev().attr( "name" ); - if (!this.id && previousSiblingName) { - this.id = $(this).attr( "id", previousSiblingName.replace(/\./g, "-") ); - } - return this.id; - }), output = $(this); - - if (!headers.length || headers.length < settings.minimumHeaders || !output.length) { - return; - } - - var get_level = function(ele) { return parseInt(ele.nodeName.replace("H", ""), 10); } - var highest_level = headers.map(function(_, ele) { return get_level(ele); }).get().sort()[0]; - - var level = get_level(headers[0]), - this_level, - html = settings.title + ' <'+settings.listType+' class="jekyll-toc">'; - - var back_to_top = function(id) { - return ''+settings.backToTopText+''; - } - - var link_here = function(id) { - return ''+settings.linkHereText+''; - } - - function fixedEncodeURIComponent (str) { - return encodeURIComponent(str).replace(/[!'()*]/g, function(c) { - return '%' + c.charCodeAt(0).toString(16); - }); - } - - function force_update_hash(hash) { - if ( window.location.hash == hash ) { - window.location.hash = ''; - } - window.location.hash = hash; - - } - - $(headers).each(function(_, header) { - this_level = get_level(header); - var header_id = $(header).attr('id'); - if (this_level === level) // same level as before; same indenting - html += "
        • " + header.innerHTML + ""; - else if (this_level <= level){ // higher level than before; end parent ol - for(i = this_level; i < level; i++) { - html += "
          • " - } - html += "
        • " + header.innerHTML + ""; - } - else if (this_level > level) { // lower level than before; expand the previous to contain a ol - for(i = this_level; i > level; i--) { - html += "<"+settings.listType+">
        • " - } - html += "" + header.innerHTML + ""; - } - level = this_level; // update for the next one - - // add links at the end (so we don't pulute header.innerHTML) - $(header).addClass('jekyll-toc-top-level-header').wrapInner('').append( link_here(header_id) ); - if (settings.backToTop) { - switch(settings.backToTopDisplay){ - case 'highest': - if ( this_level === highest_level ) { - $(header).append( back_to_top(settings.backToTopId) ); - } - break; - case 'always': - default: - $(header).append( back_to_top(settings.backToTopId) ); - } - } - - if (settings.linkHeader) { - $(header).addClass('jekyll-toc-header'); - $(header).children('span.jekyll-toc-wrapper').on( 'click', function( ) { - force_update_hash(fixedEncodeURIComponent(header_id)); - }); - } - }); - - html += ""; - - output.html(html) - }; -})(jQuery); \ No newline at end of file diff --git a/learning.markdown b/learning.markdown index 3454b54e3..c78d1ba5a 100644 --- a/learning.markdown +++ b/learning.markdown @@ -2,97 +2,176 @@ title: "Learning resources" section: learning layout: default +image: /images/social/elixir-og-card.jpg --- -# {{ page.title }} +# Learning {% include toc.html %} -Our website provides a [Getting Started guide](/getting-started/introduction.html) to learn more about Elixir's foundation and explore how to build projects with [Mix and OTP](getting-started/mix-otp/introduction-to-mix.html). +Elixir's official documentation includes a [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more about Elixir's foundations. Later on, it explores how to build projects with [Mix and OTP](https://hexdocs.pm/elixir/introduction-to-mix.html). Elixir also includes [extensive API documentation](/docs.html). -The Elixir Community has also produced plenty of resources to explore Elixir from different backgrounds and other perspectives. We are sure you will find a resource that follows your pace and interests. +The Elixir Community has also produced plenty of resources to learn the language from different backgrounds and other perspectives. We list some of them below. We are sure you will find a resource that suits your pace and goals. -## Learn Elixir +## Books -

          Programming Elixir 1.3

          +

          Elixir in Action

          -Programming Elixir cover +Elixir in Action cover -Explore functional programming without the academic overtones (tell me about monads just one more time). Create concurrent applications, but get them right without all the locking and consistency headaches. Meet Elixir, a modern, functional, concurrent language built on the rock-solid Erlang VM. +Elixir in Action is a tutorial book that aims to bring developers new to Elixir and Erlang to the point where they can develop complex systems on their own. No knowledge about Elixir, Erlang, or functional programming is required, but it is assumed that a reader has a few years of production experience using mainstream OO languages, for example C#, Java, Python, or Ruby. -Elixir’s pragmatic syntax and built-in support for metaprogramming will make you productive and keep you interested for the long haul. Maybe the time is right for the Next Big Thing. Maybe it’s Elixir. This book is the introduction to Elixir for experienced programmers, completely updated for Elixir 1.3. +The book starts with a basic introduction to the Elixir language and functional programming idioms. The central part of the book deals with Erlang VM and OTP, discussing topics such as concurrent programming, fault-tolerance, and distributed systems. Finally, you'll learn how to package your code into components, create a standalone deployable release, and troubleshoot the running system. The theory is demonstrated through a simplistic example that is gradually expanded throughout the book into a fully standalone releasable system.
          -

          Elixir in Action

          +

          Programming Elixir 1.6

          -Elixir in Action cover +Programming Elixir cover -Elixir in Action is a tutorial book that aims to bring developers new to Elixir and Erlang to the point where they can develop complex systems on their own. No knowledge about Elixir, Erlang, or functional programming is required, but it is assumed that a reader has a few years of production experience using mainstream OO languages, for example C#, Java, Python, or Ruby. +This book is the introduction to Elixir for experienced programmers, completely updated for Elixir 1.6 and beyond. Explore functional programming without the academic overtones (tell me about monads just one more time). Create concurrent applications, but get them right without all the locking and consistency headaches. -The book starts with a basic introduction to the Elixir language and functional programming idioms. The central part of the book deals with Erlang VM and OTP, discussing topics such as concurrent programming, fault-tolerance, and distributed systems. Finally, you'll learn how to package your code into components, create a standalone deployable release, and troubleshoot the running system. The theory is demonstrated through a simplistic example that is gradually expanded throughout the book into a fully standalone releasable system. +Meet Elixir, a modern, functional, concurrent language built on the rock-solid Erlang VM. Elixir's pragmatic syntax and built-in support for metaprogramming will make you productive and keep you interested for the long haul. Maybe the time is right for the Next Big Thing. Maybe it's Elixir. + +
          + +

          Adopting Elixir

          + +Programming Elixir cover + +Adoption is more than programming. Elixir is an exciting new language, but to successfully get your application from start to finish, you're going to need to know more than just the language. You need the case studies and strategies in this book. + +Learn the best practices for the whole life of your application, from design and team-building, to managing stakeholders, to deployment and monitoring. Go beyond the syntax and the tools to learn the techniques you need to develop your Elixir application from concept to production. + +
          + +

          Joy of Elixir

          + +Joy of Elixir + +Joy of Elixir is a gentle introduction to programming, aimed at people who already know some things about computers, but who have little-to-no programming experience. + +This book will teach you the core concepts of the Elixir programming language in a fun and enjoyable way. If you're completely new to programming and you want to learn how to make a computer do things using the power of programming and you want to experience some joy while doing it, then read this book! + +
          + +

          Learn Functional Programming With Elixir

          + +Learn Functional Programming with Elixir cover + +Elixir's straightforward syntax and this guided tour give you a clean, simple path to learn modern functional programming techniques. No previous functional programming experience required! This book walks you through the right concepts at the right pace, as you explore immutable values and explicit data transformation, functions, modules, recursive functions, pattern matching, high-order functions, polymorphism, and failure handling, all while avoiding side effects. Don't board the Elixir train with an imperative mindset! To get the most out of functional languages, you need to think functionally. This book will get you there. + +
          + +

          The Toy Robot Walkthrough

          + +Toy Robot + +The Toy Robot is a common interview exercise for new programmers. This short book will take you through how to implement it in Elixir in a BDD-style, with some great explanations and imagery along the way. + +If you're a new Elixir developer who's gone through some basic Elixir tutorials and you're looking for the next thing to build your skills, this book is a great start. It covers the Toy Robot exercise from start to finish, testing with Elixir features such as ExUnit and Doctests along the way. + +
          + +

          Elixir Succinctlyfree

          + +Elixir Succinctly + +Elixir Succinctly is a free ebook to start learning Elixir. It covers the installation and the first steps with the language and the syntax. It then describes the Erlang/OTP platform, describing messages, processes, and GenServer. The final part covers the building of a sample Elixir application.
          -

          Introducing Elixir

          +## In-depth books + +

          Metaprogramming Elixir

          + +Metaprogramming Elixir cover + +Write code that writes code with Elixir macros. Macros make metaprogramming possible and define the language itself. In this book, you'll learn how to use macros to extend the language with fast, maintainable code and share functionality in ways you never thought possible. You'll discover how to extend Elixir with your own first-class features, optimize performance, and create domain-specific languages. + +
          -Introducing Elixir cover +

          Designing Elixir Systems with OTP

          -Elixir is an excellent language if you want to learn about functional programming, and with this hands-on introduction, you’ll discover just how powerful and fun Elixir can be. This language combines the robust functional programming of Erlang with a syntax similar to Ruby, and includes powerful features for metaprogramming. +Designing Elixir Systems with OTP cover -This book shows you how to write simple Elixir programs by teaching one skill at a time. Once you pick up pattern matching, process-oriented programming, and other concepts, you’ll understand why Elixir makes it easier to build concurrent and resilient programs that scale up and down with ease. +You know how to code in Elixir; now learn to think in it. Learn to design libraries with intelligent layers that shape the right data structures, flow from one function into the next, and present the right APIs. Embrace the same OTP that's kept our telephone systems reliable and fast for over 30 years. Move beyond understanding the OTP functions to knowing what's happening under the hood, and why that matters. Using that knowledge, instinctively know how to design systems that deliver fast and resilient services to your users, all with an Elixir focus.
          -

          The Little Elixir and OTP Guidebook

          +

          Concurrent Data Processing in Elixir

          -The Little Elixir and OTP Guidebook cover +Concurrent Data Processing cover -The Little Elixir & OTP Guidebook gets you started programming applications with Elixir and OTP. You begin with a quick overview of the Elixir language syntax, along with just enough functional programming to use it effectively. Then, you'll dive straight into OTP and learn how it helps you build scalable, fault-tolerant and distributed applications through several fun examples. Come rediscover the joy of programming with Elixir and remember how it feels like to be a beginner again. +Learn different ways of writing concurrent code in Elixir and increase your application’s performance, without sacrificing scalability or fault-tolerance. Most projects benefit from running background tasks and processing data concurrently, but the world of OTP and various libraries can be challenging. Which Supervisor and what strategy to use? What about GenServer? Maybe you need back-pressure, but is GenStage, Flow, or Broadway a better choice? You will learn everything you need to know to answer these questions, start building highly concurrent applications in no time, and write code that’s not only fast, but also resilient to errors and easy to scale.
          -

          Elixir Cookbook

          +

          Erlang in Angerfree

          -Elixir Cookbook cover +Erlang in Anger cover -This book is a set of recipes grouped by topic that acts as a reference to get ideas from or to quickly search for a solution to a problem. You will begin by launching an IEx session and using it to test some ideas. Next, you will perform various operations like loading and compiling modules, inspecting your system, generating a supervised app, and so on. Furthermore, you will be introduced to immutability, working with data structures, performing pattern matching, and using stream modules to generate infinite data sequences. You will learn about everything from joining strings to determining the word frequency in text. With respect to modules and functions, you will also discover how to load code from other modules and use guards and pattern matching in functions. +This book intends to be a little guide about how to be the Erlang medic in a time of war. It is first and foremost a collection of tips and tricks to help understand where failures come from, and a dictionary of different code snippets and practices that helped developers debug production systems that were built in Erlang.
          -

          Elixir School

          +## Courses + +

          Elixir Schoolfree

          Elixir School -Elixir-School is an open and community driven effort inspired by Twitter’s Scala School. The site’s content consists of peer-reviewed lessons on various Elixir topics that range in difficulty. The lessons are currently available in over 10 languages to help make programming Elixir more accessible to non-English speakers. +Elixir-School is an open and community driven effort inspired by Twitter's Scala School. The site's content consists of peer-reviewed lessons on various Elixir topics that range in difficulty. The lessons are currently available in over 10 languages to help make programming Elixir more accessible to non-English speakers.
          -

          Take Off With Elixir

          +

          Pragmatic Studio's Elixir/OTP Course

          -Red:4 +Pragmatic Studio's Elixir/OTP Course +Put Elixir and OTP into action as you build a concurrent, fault-tolerant application from scratch in this 6-hour video course from The Pragmatic Studio. By developing a real app with real code, you'll gain practical experience putting all the pieces together to craft applications the Elixir/OTP way. -Red:4 is a fictional aerospace startup that needs your help! Through a book, a video, or both you can learn Elixir the fun way by immersing yourself in an on-the-job style set of problems. For instance, you will set up a project to calculate escape velocity for each of the planets in our solar system, learning pattern matching and language basics along the way. You'll move on to orbital mechanics while learning how to debug and refactor your code for clarity and meaning. You'll learn list basics and data storage techniques as you build a solar flare tracking system. Finally, you'll dive into OTP using Ecto and PostgreSQL - all in an effort to overwhelm our internal systems with the power of Elixir! +The first half of the course focuses on core Elixir facets, principles, and techniques. In the second half, we go beyond the basics and focus on what sets Elixir/OTP apart: concurrent processes, the actor model, OTP behaviors, and fault recovery. + +If you're new to Elixir, you'll get step-by-step guidance in an engaging format you won't find anywhere else. If you have experience with Elixir, you'll gain a deeper understanding of things you've been taking for granted and fill in any knowledge gaps.
          +

          grox.io's Elixir Course

          -## Screencasts +grox.io's Multi-Format Elixir Course + +Learning complex concepts like programming languages is best with multiple formats. Groxio's learning method embraces an interactive mini-book for beginners, video overviews for novices, an online book for presenting higher level concepts, videos with live coding to simulate advanced pair programming through projects meaningful to beginners and experts. + +The Elixir module is a full program with a book with 80 pages, 8 videos, dozens of exercises, and two full test-first projects. Beginners can learn the language, and experts can fill in typical blind spots like writing sigils, building macros, and using streams. + +
          + +

          grox.io's OTP Course

          + +grox.io's Multi-Format OTP Course + +This course teaches OTP from a design perspective by showing a system for breaking projects into layers. This course builds on those layering concepts with a 60 page book, 12 videos, projects, and curated links to go into deeper detail for tricky OTP concepts. + +Understand how back-pressure works, step inside supervision trees, and learn to build your dynamic supervisors. Learn OTP for the first time, or solidify your intuition by building the base concepts by reading a book, watching videos, and working through guided projects using Groxio's blend of media, designed to take you from novice to expert. -

          Elixir Sips

          +
          + +

          ThinkingElixir.com's Pattern Matching Coursefree

          -ElixirSips cover +ThinkingElixir.com's Free Pattern Matching Course -Elixir Sips is a screencast series that provides 2 short videos - typically from 2 to 7 minutes, but occasionally much longer - each week. The videos consist of various topics, ranging from exploring a module in the standard library to trying out a new project to building a web-based Tetris game from scratch. +Pattern matching is a really powerful language feature. It is built in to almost every corner or Elixir. In order for you to even read Elixir code and follow along, you have to understand Pattern Matching. -The intended audience ranges from someone entirely new to the Elixir language, to experienced developers that want to get a broad range of topics to think about from time to time. +Once you "get" pattern matching, it feels like a super power. Pattern Matching makes new patterns of coding possible. You start to unlearn some of the patterns you've used in other languages because now you can create even clearer and more elegant code than was possible before! + +The course covers getting setup with Elixir, the data types, how to pattern match each of them, and more! An included TDD project helps you easily apply what you're learning.
          @@ -102,36 +181,123 @@ The intended audience ranges from someone entirely new to the Elixir language, t LearnElixir.tv cover -LearnElixir.tv is a screencast series which provides in-depth, step-by-step videos about Elixir's main features. Videos range from 7 to 15 minutes in length, and are posted weekly. +LearnElixir.tv is a video course which provides in-depth, step-by-step videos about Elixir's main features. Videos range from 7 to 15 minutes in length. It's intended to help beginners get familiar with all of Elixir's features by building their knowledge incrementally. Experienced Elixir developers might also learn a trick or two.
          -## In-depth Resources +

          Educative.io's Metaprogramming in Elixir Course

          + + + Educative.io's Metaprogramming in Elixir Course + + +Get introduced to the concept of metaprogramming. Learn how to level up your programming skills by discovering the full potential of the macro system in Elixir. Understand the ins and outs of metaprogramming at a fundamental level and write incredible libraries by doing more with less code. + +
          + +

          Learn-Elixir.dev

          + + + Learn-Elixir.dev cover + -

          Études for Elixir

          +At your own pace, progress through 132 videos, 30 quizzes, 9 assignments, and 2 major projects of your choosing to demonstrate your working knowledge of a production-level quality of code. You’ll learn Syntax/Fundamentals, REST, GraphQL with Absinthe, Phoenix, OTP, Testing, Ecto, architecture, how to scale, how to go distributed, and more… -Études for Elixir +Join our weekly live coaching sessions every Wednesday at 12:00-noon pacific time to close any gaps in knowledge and get specific answers to your specific questions! -Études for Elixir is an open source book of programming exercises. Each one has been designed to provide practice material for a particular Elixir programming concept. The topics are keyed to the chapters in "Introducing Elixir," but they are general enough to be used with any Elixir book. Études for Elixir can be read free online. +Found your start-up, migrate a codebase, build that app! Our midterm and final projects create a great win-win scenario for you to complete your personal project while mastering the language. Or if you’re more career-focused we can offer referrals, recommendations, or references and host your completed projects on GitHub to show hiring managers your proficiency programming Elixir.
          -

          Metaprogramming Elixir

          +

          TechSchoolfree

          -Metaprogramming Elixir cover + + TechSchool + -Write code that writes code with Elixir macros. Macros make metaprogramming possible and define the language itself. In this book, you’ll learn how to use macros to extend the language with fast, maintainable code and share functionality in ways you never thought possible. You’ll discover how to extend Elixir with your own first-class features, optimize performance, and create domain-specific languages. +TechSchool is an open-source platform that teaches programming through free YouTube videos and other websites. The goal is to make technology education accessible to everyone. It includes several Elixir courses and a complete Fullstack Elixir + Phoenix Bootcamp.
          -

          Erlang in Anger

          +## Screencasts -Erlang in Anger cover +

          ElixirStreamsfree

          -This book intends to be a little guide about how to be the Erlang medic in a time of war. It is first and foremost a collection of tips and tricks to help understand where failures come from, and a dictionary of different code snippets and practices that helped developers debug production systems that were built in Erlang. + + elixir streams cover + + +ElixirStreams provides free video tips (under 3 mins!) covering a variety of +Elixir and Phoenix topics. The videos help you sharpen the saw as you learn +about new tools and tricks, and they keep you up to date with the latest +developments in the language. + +
          + +

          ElixirCasts.iofree

          + + + elixircasts.io cover + + +ElixirCasts is a collection of simple screencasts that cover a wide range of Elixir and Phoenix topics. Each episode tackles a specific problem or explores a new library, demystifying it in a language that's easy to understand. + +Episodes range from beginner focused to more moderate and advanced topics. Come build your knowledge of Elixir with us, one episode at a time. + +
          + +

          Alchemist Campfree

          + +Alchemist Camp cover + +Alchemist Camp is the largest producer of free Elixir screencasts and has dozens of hours of screencasts on YouTube. The videos are often longer-form and focused around projects, such as building a small Phoenix clone, or an OTP worker to regularly collect statistics from multiple APIs. Content is driven by viewer request. + +Alchemist Camp is aimed at people who have some web development experience and want to ship real-world projects in Elixir. + +
          + +## Other resources + +

          Elixir Flashcards

          + +Elixir Flashcards + +Elixir flashcards are a set of beautifully crafted, professionally printed, poker sized flashcards to help you master the Elixir language. + +Flashcards are a great way to highlight knowledge gaps, identify misconceptions or false beliefs, and help you memorise key concepts. +When used in groups or teams, flashcards can help spark interesting discussions, and help bring people together to learn in a fun way by playing games. + +Combined with books, tutorials and screencasts, using flashcards is the killer combination to master Elixir. + +
          + +

          Elixir Koansfree

          + +Elixir Koans + +Elixir koans is a fun, easy way to get started with the Elixir programming language. It is an idiomatic tour of the language. + +
          + +

          Exercismfree

          + + Exercism Elixir track + +Exercism is an open source platform that provides free practice and mentoring in many languages, including Elixir. +It features exercises of varying difficulty, from string processing to using OTP, that are mentored by volunteers. +Once you have completed an exercise you can also view other students' solutions. + +
          + +

          Running in Production Podcastfree

          + +Running in Production Podcast + +Running in Production is a podcast where developers and engineers talk about +running small and large Elixir / Phoenix web apps in production. + +Topics include tech stacks, success stories, lessons learned and deployment tips.
          diff --git a/opensearch.xml b/opensearch.xml index 784276e91..4e982e7c5 100644 --- a/opensearch.xml +++ b/opensearch.xml @@ -9,7 +9,7 @@ elixir-lang.org - http://elixir-lang.org/favicon.ico \ No newline at end of file diff --git a/trademarks.markdown b/trademarks.markdown new file mode 100644 index 000000000..329585587 --- /dev/null +++ b/trademarks.markdown @@ -0,0 +1,70 @@ +--- +title: "Trademarks policy" +section: trademarks +layout: default +image: /images/social/elixir-og-card.jpg +--- + +# {{ page.title }} + +This document outlines the policy for allowed usage of the "Elixir" word and the Elixir logo by other parties. + +"Elixir" and the Elixir logo are registered trademarks of the Elixir Team. The Elixir Team believes in a decentralized approach to growing the community and the ecosystem, independent of the Elixir project and the Elixir Team. + +Anyone can use the Elixir trademarks if that use of the trademark is nominative. The trademarks must not be used to disparage the project and its community, nor be used in any way to imply ownership, endorsement, or association with the Elixir project and the Elixir Team. + +You must not visually combine the Elixir logo with any other images, or change the logo in any way other than ways required by printing restrictions. If you want to create your own visual identity in relation to Elixir, you might use the shape of an unrelated "water drop" as part of your design, as seen in many community projects and initiatives. You must not combine or modify the Elixir logo. + +The Elixir logo is [available in our repository](https://github.com/elixir-lang/elixir-lang.github.com/tree/main/downloads/logos) in both vertical and horizontal versions. + +## Nominative use + +The "nominative use" (or "nominative fair use") is a legal doctrine that authorizes everyone (even commercial companies) to use or refer to the trademark of another if: + + * The product or service in question must be one not readily identifiable without use of the trademark. + + * Only so much of the mark or marks may be used as is reasonably necessary to identify the product or service. + + * The organization using the mark must do nothing that would, in conjunction with the mark, suggest sponsorship or endorsement by the trademark holder. + +Our trademarks must be used to refer to the Elixir programming language. + +## Examples of permitted use + +All examples listed next must strictly adhere to the terms outlined in the previous sections: + + * Usage of the Elixir logo to say a technology is "powered by Elixir" under nominative use. Linking back to the Elixir website, if possible, is appreciated. + + * Usage of the Elixir logo to display it as a supported technology in a service or platform. For instance, you may say "we support Elixir" and use the Elixir logo, but you may not refer to yourself as "the Elixir platform" nor imply any form of endorsement or association with Elixir. + + * Usage of the Elixir logo in non-commercial community meetups, in presentations, and in courses when referring to the language and its ecosystem under nominative use. + + * Usage of the Elixir logo in non-commercial swag (stickers, t-shirts, mugs, etc) to promote the Elixir programming language. The Elixir marks must be the only marks featured in the product. You need permission to make swag that include Elixir and other third party marks in them. + + * Inclusion of the Elixir logo in non-commercial icon sets. Use of the Elixir icons must still adhere to Elixir's trademark policies. + + * Usage of the "Elixir" word in book titles, meetups, conferences, and podcasts. You must not use the word to imply uniqueness or endorsement from the Elixir team. "The Elixir book" and "The Elixir podcast" are not permitted. "Elixir in Action", "Thinking Elixir", and "Kraków Elixir User Group" are valid examples already in use today. + + * Usage of the "Elixir" word in the names of freely distributed software and hardware products is allowed when referring to use with or suitability for the Elixir programming language, such as wxElixir, Elixirsense, etc. If the product includes the Elixir programming language itself, then [you must also respect its license](https://github.com/elixir-lang/elixir/blob/main/LICENSE). + +## Examples of not permitted use + +Here is a non-exhaustive list of non permitted uses of the marks: + + * Usage of the Elixir logo in book covers, conferences, and podcasts. + + * Usage of the Elixir logo as the mark of third party projects, even in combination with other marks. + + * Naming any company or product after Elixir, such as "The Elixir Hosting", "The Elixir Consultants", etc. + +## Examples that require permission + +Here are some examples that may be granted permission upon request: + + * Selling merchandise (stickers, t-shirts, mugs, etc). + +You can request permission by emailing trademarks@elixir-lang.org. + +## Important note + +Nothing in this page shall be interpreted to allow any third party to claim any association with the Elixir project and the Elixir Team, or to imply any approval or support by the Elixir project and the Elixir Team for any third party products, services, or events.