diff --git a/.gitignore b/.gitignore index 16182c5d0..05dfac0b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ -/_site \ No newline at end of file +/_site +/.jekyll-metadata +/downloads/cheatsheets/*.aux +/downloads/cheatsheets/*.log +/.bundle +/vendor \ No newline at end of file diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 000000000..944880fa1 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.2.0 diff --git a/CNAME b/CNAME index 615972c11..e9fe306fb 100644 --- a/CNAME +++ b/CNAME @@ -1 +1 @@ -elixir-lang.org +elixir-lang.org \ No newline at end of file diff --git a/Gemfile b/Gemfile new file mode 100644 index 000000000..f24d05f4a --- /dev/null +++ b/Gemfile @@ -0,0 +1,8 @@ +source 'https://rubygems.org' + +gem 'github-pages' +gem 'json', '>= 2.0.0' +gem 'webrick', '>= 1.8' +gem 'csv', '~> 3.3' +gem 'base64', '~> 0.2.0' +gem 'bigdecimal', '~> 3.1' diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 000000000..f77a0774e --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,270 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.7.2) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) + base64 (0.2.0) + bigdecimal (3.1.9) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.11.1) + colorator (1.1.0) + commonmarker (0.23.10) + concurrent-ruby (1.2.2) + csv (3.3.3) + dnsruby (1.61.9) + simpleidn (~> 0.1) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + ethon (0.16.0) + ffi (>= 1.15.0) + eventmachine (1.2.7) + execjs (2.8.1) + faraday (2.7.4) + faraday-net_http (>= 2.0, < 3.1) + ruby2_keywords (>= 0.0.4) + faraday-net_http (3.0.2) + ffi (1.15.5) + forwardable-extended (2.6.0) + gemoji (3.0.1) + github-pages (228) + github-pages-health-check (= 1.17.9) + jekyll (= 3.9.3) + jekyll-avatar (= 0.7.0) + jekyll-coffeescript (= 1.1.1) + jekyll-commonmark-ghpages (= 0.4.0) + jekyll-default-layout (= 0.1.4) + jekyll-feed (= 0.15.1) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.13.0) + jekyll-include-cache (= 0.2.1) + jekyll-mentions (= 1.6.0) + jekyll-optional-front-matter (= 0.3.2) + jekyll-paginate (= 1.1.0) + jekyll-readme-index (= 0.3.0) + jekyll-redirect-from (= 0.16.0) + jekyll-relative-links (= 0.6.1) + jekyll-remote-theme (= 0.4.3) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.8.0) + jekyll-sitemap (= 1.4.0) + jekyll-swiss (= 1.0.0) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) + jekyll-titles-from-headings (= 0.5.3) + jemoji (= 0.12.0) + kramdown (= 2.3.2) + kramdown-parser-gfm (= 1.1.0) + liquid (= 4.0.4) + mercenary (~> 0.3) + minima (= 2.5.1) + nokogiri (>= 1.13.6, < 2.0) + rouge (= 3.26.0) + terminal-table (~> 1.4) + github-pages-health-check (1.17.9) + addressable (~> 2.3) + dnsruby (~> 1.60) + octokit (~> 4.0) + public_suffix (>= 3.0, < 5.0) + typhoeus (~> 1.3) + html-pipeline (2.14.3) + activesupport (>= 2) + nokogiri (>= 1.4) + http_parser.rb (0.8.0) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + jekyll (3.9.3) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (>= 0.7, < 2) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + jekyll-avatar (0.7.0) + jekyll (>= 3.0, < 5.0) + jekyll-coffeescript (1.1.1) + coffee-script (~> 2.2) + coffee-script-source (~> 1.11.1) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.4.0) + commonmarker (~> 0.23.7) + jekyll (~> 3.9.0) + jekyll-commonmark (~> 1.4.0) + rouge (>= 2.0, < 5.0) + jekyll-default-layout (0.1.4) + jekyll (~> 3.0) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-gist (1.5.0) + octokit (~> 4.2) + jekyll-github-metadata (2.13.0) + jekyll (>= 3.4, < 5.0) + octokit (~> 4.0, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-mentions (1.6.0) + html-pipeline (~> 2.3) + jekyll (>= 3.7, < 5.0) + jekyll-optional-front-matter (0.3.2) + jekyll (>= 3.0, < 5.0) + jekyll-paginate (1.1.0) + jekyll-readme-index (0.3.0) + jekyll (>= 3.0, < 5.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-remote-theme (0.4.3) + addressable (~> 2.0) + jekyll (>= 3.5, < 5.0) + jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) + rubyzip (>= 1.3.0, < 3.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-swiss (1.0.0) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.6.0) + jekyll (> 3.5, < 5.0) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.3) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.12.0) + gemoji (~> 3.0) + html-pipeline (~> 2.2) + jekyll (>= 3.0, < 5.0) + json (2.6.3) + kramdown (2.3.2) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.3.6) + mini_portile2 (2.8.9) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.19.0) + nokogiri (1.18.9) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.7) + racc (1.8.1) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.4.2) + rouge (3.26.0) + ruby2_keywords (0.0.5) + rubyzip (2.3.2) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) + simpleidn (0.2.1) + unf (~> 0.1.4) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) + unicode-display_width (1.8.0) + webrick (1.8.2) + +PLATFORMS + ruby + +DEPENDENCIES + base64 (~> 0.2.0) + bigdecimal (~> 3.1) + csv (~> 3.3) + github-pages + json (>= 2.0.0) + webrick (>= 1.8) + +BUNDLED WITH + 2.5.23 diff --git a/README.md b/README.md index 105415698..a091828d8 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,70 @@ -### Contents for Elixir website hosted at elixir-lang.org +This projects holds the contents for the Elixir website hosted at [elixir-lang.org](https://elixir-lang.org). -It is automatically transformed by [Jekyll](http://github.com/mojombo/jekyll) into a static site. +It is automatically transformed by [Jekyll](https://github.com/jekyll/jekyll) into a static site. -### Development +## Contributing -Development is done in the `next` branch, while `master` contains the docs and guides for the latest stable release. +#### 1. Fork and clone this repository -### Contributing to the blog +[Fork this +repository](https://github.com/elixir-lang/elixir-lang.github.com/fork) and +clone your fork. If you don't know what forking means or don't know how to do +it, nice instructions are available +[here](https://help.github.com/articles/fork-a-repo/). - Create a new file inside `_posts/YYYY-MM-DD-post-title.markdown` following the template: +#### 2. Install Ruby - --- - layout: post - title: - author: - category: - excerpt: - --- +This website is compiled into a static website using +[Jekyll](http://jekyllrb.com), a static-site generator written in Ruby. To +install Ruby you can follow [this +guide](https://www.ruby-lang.org/en/documentation/installation/). To check that +Ruby is installed correctly, run `ruby --version` in your shell; it should be +`1.9.3` or later. - Body text goes here... +#### 3. Install Bundler to manage dependencies - Or use `_bin/newpost` to bootstrap a new post file: +[Bundler](http://bundler.io) handles Ruby dependencies. To install it, simply +run: - export EDITOR=vim; _bin/newpost 'Post title' +```bash +$ gem install bundler +``` -### Contributing improvements or bug fixes +Once you have installed it, `cd` into the local clone of your fork and run: -1. Fork elixir-lang.github.com +```bash +$ bundle install +``` -2. Make your changes +to download and install the necessary dependencies. -3. Test it locally +#### 4. Run Jekyll - You need to install `jekyll`, `jekyll-redirect-from` and `redcarpet` +In order to run a development server (with live-reloading on) just run: - ```shell - $ gem install jekyll jekyll-redirect-from redcarpet - $ jekyll serve # check localhost:4000 - ``` +```bash +$ bundle exec jekyll serve +``` -4. Send a pull-request for your changes. +The generated site will be available at [http://localhost:4000](http://localhost:4000). You can stop the +server with Ctrl+C. -### License +#### 5. Make your changes and push them -* "Elixir" and the Elixir logo are copyrighted to [Plataformatec](http://plataformatec.com.br/). You may not reuse anything therein without permission. +Now you're ready to make your changes! Be sure to test the changes locally using +the development server. Once you're done with your changes, push those changes +to your fork and then [submit a **pull +request**](https://help.github.com/articles/using-pull-requests/). For a nice +wrap-up on how to open a good pull request have a look at the [Elixir +contributing +guide](https://github.com/elixir-lang/elixir/#contributing). + +## License + +* "Elixir" and the Elixir logo are registered trademarks of the Elixir team. See [our trademark policy](https://elixir-lang.org/trademarks). * The HTML and CSS are copyrighted to [AlienWp](http://alienwp.com/) under [GPL license, version 2](http://www.gnu.org/licenses/old-licenses/gpl-2.0.html). * The Social Icons are copyrighted to [Xeloader](http://xeloader.deviantart.com/art/Socialis-2-Freebie-213292616). -* The written textual contents available in the guides and blog are licensed under Apache 2.0. - -* The available docs are licensed under the same license as their projects. +* The written textual contents available in the guides and blog are licensed under [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). diff --git a/_bin/newpost b/_bin/newpost deleted file mode 100755 index 1092a5c78..000000000 --- a/_bin/newpost +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env ruby - -unless ARGV[0] - puts 'Usage: _bin/newpost "Post Title"' - exit(-1) -end - -date_prefix = Time.now.strftime("%Y-%m-%d") -post_name = ARGV.join ' ' -permalink = post_name.strip.downcase.gsub(/[^a-zA-Z0-9]/, '-') -filename = "#{date_prefix}-#{permalink}.markdown" -post = File.expand_path("../_posts/#{filename}", File.dirname(__FILE__)) - -header = <<-END ---- -layout: post -title: #{post_name} -author: -category: -excerpt: ---- - -Body text goes here... -END - -File.open(post, 'w') do |f| - f << header -end - -if editor = ENV['EDITOR'] do - system(editor, post) -else - puts "Post created at: #{post}" -end - -exit(0) diff --git a/_config.yml b/_config.yml index 78201b070..635db0b94 100644 --- a/_config.yml +++ b/_config.yml @@ -1,5 +1,29 @@ -markdown: redcarpet -pygments: true -permalink: /blog/:year/:month/:day/:title -redcarpet: - extensions: ['with_toc_data'] +highlighter: rouge +permalink: none +kramdown: + input: GFM + hard_wrap: false +repository: elixir-lang/elixir-lang.github.com +url: https://elixir-lang.org +exclude: + - _build/ + - vendor/ + - Gemfile + - Gemfile.lock + - CNAME + - .gitignore + - README.md + - js/toc/README.md +plugins: + - jemoji + - jekyll-sitemap + - jekyll-redirect-from + - jekyll-seo-tag +defaults: + - scope: + path: "" + type: posts + values: + layout: post + permalink: /blog/:year/:month/:day/:title/ + image: /images/social/elixir-og-card.jpg diff --git a/_data/elixir-versions.yml b/_data/elixir-versions.yml new file mode 100644 index 000000000..468bdabbf --- /dev/null +++ b/_data/elixir-versions.yml @@ -0,0 +1,88 @@ +stable: v1_19 + +v1_19: + name: v1.19 + minimum_otp: 26.0 + recommended_otp: 28.1 + otp_versions: [28, 27, 26] + version: 1.19.4 + +v1_18: + name: v1.18 + minimum_otp: 26.0 + recommended_otp: 27.3.4 + otp_versions: [27, 26, 25] + version: 1.18.4 + +v1_17: + name: v1.17 + minimum_otp: 25.0 + recommended_otp: 27.1.2 + otp_versions: [27, 26, 25] + version: 1.17.3 + +v1_16: + name: v1.16 + minimum_otp: 24.0 + otp_versions: [26, 25, 24] + version: 1.16.3 + +v1_15: + name: v1.15 + minimum_otp: 24.0 + otp_versions: [26, 25, 24] + version: 1.15.8 + +v1_14: + name: v1.14 + minimum_otp: 23.0 + otp_versions: [25, 24, 23] + version: 1.14.5 + +v1_13: + name: v1.13 + minimum_otp: 22.0 + otp_versions: [24, 23, 22] + version: 1.13.4 + +v1_12: + name: v1.12 + minimum_otp: 22.0 + otp_versions: [24, 23, 22] + version: 1.12.3 + +v1_11: + name: v1.11 + minimum_otp: 21.0 + otp_versions: [23, 22, 21] + version: 1.11.4 + +v1_10: + name: v1.10 + minimum_otp: 21.0 + otp_versions: [22, 21] + version: 1.10.4 + +v1_9: + name: v1.9 + minimum_otp: 20.0 + otp_versions: [22, 21, 20] + version: 1.9.4 + +v1_8: + name: v1.8 + minimum_otp: 20.0 + otp_versions: [22, 21, 20] + version: 1.8.2 + +v1_7: + name: v1.7 + minimum_otp: 19.0 + otp_versions: [21, 20, 19] + version: 1.7.4 + +v1_6: + name: v1.6 + minimum_otp: 19.0 + otp_versions: [21, 20, 19] + version: 1.6.6 diff --git a/_includes/bottom.html b/_includes/bottom.html index da7ab3085..bfc4fb987 100644 --- a/_includes/bottom.html +++ b/_includes/bottom.html @@ -1,36 +1,12 @@ -
-
+
- - - - - - - diff --git a/_includes/categories-list.html b/_includes/categories-list.html index 8056b1d0d..61d4e22c0 100644 --- a/_includes/categories-list.html +++ b/_includes/categories-list.html @@ -1,8 +1,16 @@

Blog Categories

diff --git a/_includes/code-editor-support.html b/_includes/code-editor-support.html deleted file mode 100644 index 190d219f5..000000000 --- a/_includes/code-editor-support.html +++ /dev/null @@ -1,10 +0,0 @@ -
-

Code editor support

- -
diff --git a/_includes/events.html b/_includes/events.html new file mode 100644 index 000000000..e8ed467c5 --- /dev/null +++ b/_includes/events.html @@ -0,0 +1,4 @@ + diff --git a/_includes/important-links.html b/_includes/important-links.html index fcce77aa5..c4d5c0702 100644 --- a/_includes/important-links.html +++ b/_includes/important-links.html @@ -1,24 +1,63 @@
-

Join the Community

+

Important links

+
+ +
+
Watch the Elixir
mini-documentary!
+
+ +
+ +{% include events.html %} +
-

Important links

+

Join the Community

-{% include learning-resources.html %} - -{% include code-editor-support.html %} - -{% include sponsors.html %} +
+ Join the Erlang Ecosystem Foundation +
diff --git a/_includes/learning-resources.html b/_includes/learning-resources.html deleted file mode 100644 index c9258d28b..000000000 --- a/_includes/learning-resources.html +++ /dev/null @@ -1,10 +0,0 @@ -
-

Learning resources

- -
diff --git a/_includes/pagination.html b/_includes/pagination.html index 111c62b82..7e5dd07cb 100644 --- a/_includes/pagination.html +++ b/_includes/pagination.html @@ -1,7 +1,3 @@ - - diff --git a/_includes/sponsors.html b/_includes/sponsors.html deleted file mode 100644 index 97d41ae9a..000000000 --- a/_includes/sponsors.html +++ /dev/null @@ -1,6 +0,0 @@ -
-

Sponsors

- -
\ No newline at end of file diff --git a/_includes/toc.html b/_includes/toc.html new file mode 100644 index 000000000..d4e82e047 --- /dev/null +++ b/_includes/toc.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/_includes/top.html b/_includes/top.html index 3d1865f8a..13ecc8304 100644 --- a/_includes/top.html +++ b/_includes/top.html @@ -1,14 +1,25 @@ - + - - - {% if page.title %}{{ page.title }} - {% endif %}Elixir - - - - - + + + + {% if page.title %}{{ page.title }} - {% endif %}The Elixir programming language + + + + + + + + + + + + + + + {% seo title=false %} @@ -16,29 +27,22 @@
diff --git a/_layouts/blog.html b/_layouts/blog.html index 72a180dbc..92a81743d 100644 --- a/_layouts/blog.html +++ b/_layouts/blog.html @@ -7,7 +7,7 @@
diff --git a/_layouts/redirect.html b/_layouts/redirect.html new file mode 100644 index 000000000..464f54e82 --- /dev/null +++ b/_layouts/redirect.html @@ -0,0 +1,14 @@ + + + + + + + + +

Redirecting...

+ Click here if you are not redirected. + + + + diff --git a/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown b/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown index 82b076aee..591610198 100644 --- a/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown +++ b/_posts/2012-04-24-a-peek-inside-elixir-s-parallel-compiler.markdown @@ -1,12 +1,13 @@ --- layout: post title: A peek inside Elixir's Parallel Compiler -author: José Valim +authors: +- José Valim category: Internals -excerpt: Today, a parallel compiler just landed in Elixir master. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. +excerpt: Today, a parallel compiler just landed in Elixir main. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. --- -Today, a parallel compiler just landed in Elixir master. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. +Today, a parallel compiler just landed in Elixir main. The goal of the parallel compiler is to compile files in parallel, automatically detecting dependencies between files. In this blog post, we are going to take a peek into the parallel compiler internals and learn more about Erlang and Elixir in the process. ## Process-based serial compilation @@ -14,7 +15,7 @@ The idea of the parallel compiler is very simple: for each file we want to compi In Elixir, we could write this code as follows: - def spawn_compilers([current|files], output) do + def spawn_compilers([current | files], output) do parent = Process.self() child = spawn_link(fn -> :elixir_compiler.file_to_path(current, output) @@ -27,12 +28,12 @@ In Elixir, we could write this code as follows: :erlang.raise(:error, reason, where) end end - + def spawn_compilers([], _output) do :done end -In the first line, we define a function named `spawn_compilers` that receives two arguments, the first is a list of files to compile and the second is a string telling us where to write the compiled file. The first argument is represented as a list with head and tail (`[current|files]`) where the top of the list is assigned to `current` and the remaining items to `files`. If the list is empty, the first clause of `spawn_compilers` is not going to match, the clause `spawn_compilers([], _output)` defined at the end will instead. +In the first line, we define a function named `spawn_compilers` that receives two arguments, the first is a list of files to compile and the second is a string telling us where to write the compiled file. The first argument is represented as a list with head and tail (`[current | files]`) where the top of the list is assigned to `current` and the remaining items to `files`. If the list is empty, the first clause of `spawn_compilers` is not going to match, the clause `spawn_compilers([], _output)` defined at the end will instead. Inside `spawn_compilers`, we first retrieve the PID of the current process with `Process.self` (remember we are talking about Erlang processes/actors and not OS processes) and then proceed to spawn a new process to execute the given function in parallel. Spawning a new process is done with the `spawn_link` function. @@ -74,7 +75,7 @@ In order to customize this process, we are going to take a look at Erlang's erro ## Custom error handler -By default, Elixir (and Erlang) code is autoloaded. This means that, if we invoke `List.delete` and the module `List` was not loaded yet, the Erlang VM is going to look into the `ebin` directory (the directory where we put compiled files) and try to load it. This process is controlled by the [`error_handler` module in Erlang](http://erlang.org/doc/man/error_handler.html) via two callback functions: `undefined_function` and `undefined_lambda`. +By default, Elixir (and Erlang) code is autoloaded. This means that, if we invoke `List.delete` and the module `List` was not loaded yet, the Erlang VM is going to look into the `ebin` directory (the directory where we put compiled files) and try to load it. This process is controlled by the [`error_handler` module in Erlang](http://www.erlang.org/doc/man/error_handler.html) via two callback functions: `undefined_function` and `undefined_lambda`. As discussed in the previous section, we want to extend the error handler to actually stop the currently running process whenever a module is not found and resume the process only after we ensure the module is compiled. To do that, we can simply define our own error handler and ask Erlang to use it. Our custom error handler is defined as follows: @@ -121,13 +122,13 @@ Notice that we have two small additions. First we store the `:elixir_parent_comp Second, our main process can now receive a new `{ :waiting, child, module }` message, so we need to extend it to account for those messages. Not only that, we need to control which PIDs we have spawned so we can notify them whenever a new module is compiled, forcing us to add a new argument to the `spawn_compilers` function. `spawn_compilers` would then be rewritten as follows: - def spawn_compilers([current|files], output, stack) do + def spawn_compilers([current | files], output, stack) do parent = Process.self() child = spawn_link(fn -> :elixir_compiler.file_to_path(current, output) send parent, { :compiled, Process.self() } end) - wait_for_messages(files, output, [child|stack]) + wait_for_messages(files, output, [child | stack]) end # No more files and stack is empty, we are done @@ -156,7 +157,7 @@ Notice we added an extra clause to `spawn_compilers` so we can properly handle t :erlang.raise(:error, reason, where) after 10_000 -> - raise "dependency on unexesting module or possible deadlock" + raise "dependency on nonexistent module or possible deadlock" end end @@ -166,7 +167,7 @@ The implementation for `wait_for_messages` is now broken into 4 clauses: * `{ :waiting, _child, _module }` - A message received every time a child process is waiting on a module to be compiled. In this scenario, all we do is spawn a new process to compile another file, ensuring compilation is never blocked; -* `{ :EXIT, _child, { reason, where } }` - The same behavior as before, it simply raises an error if any of the child processes fail; +* `{ :EXIT, _child, { reason, where } }` - The same behaviour as before, it simply raises an error if any of the child processes fail; * `after: 10_000` - This clause is going to be invoked whenever the main process does not receive a message for 10 seconds. This means a file depends on a module that does not exist (and therefore waits forever) or there is a cyclic dependency; diff --git a/_posts/2012-05-25-elixir-v0-5-0-released.markdown b/_posts/2012-05-25-elixir-v0-5-0-released.markdown index ac1f47ab7..a5701a60d 100644 --- a/_posts/2012-05-25-elixir-v0-5-0-released.markdown +++ b/_posts/2012-05-25-elixir-v0-5-0-released.markdown @@ -1,15 +1,16 @@ --- layout: post title: Elixir v0.5.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: We have finally released Elixir v0.5.0! This marks the first release since the language was rewritten. In this blog post, we will discuss what we achieved during this time and what are the next steps! --- -We have finally released [Elixir](http://elixir-lang.org/) v0.5.0! This marks the first release since the language was rewritten. In this blog post, we will discuss what we achieved during this time and what are the next steps! +We have finally released [Elixir](/) v0.5.0! This marks the first release since the language was rewritten. In this blog post, we will discuss what we achieved during this time and what are the next steps! -If you don't care about any of these, you can go straight to our [Getting Started guide](http://elixir-lang.org/getting_started/1.html). If you do, keep on reading! +If you don't care about any of these, you can go straight to our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html). If you do, keep on reading! ## Looking back @@ -19,7 +20,7 @@ At that time, Elixir attempted to be a considerable departure from Erlang and th After not feeling productive enough with that Elixir version, I have decided to take a break from Elixir to study old, new and emerging languages. The challenge was to not re-invent Erlang as a language, but how to provide the productivity and flexibility I expect from Elixir while staying a 100% compatible with Erlang. -It was around October 2011, during a short stay in San Francisco, that I came up with what would be [the foundation of Elixir's current version](http://github.com/josevalim/lego-lang) with the help of Yehuda Katz. Development of the new Elixir version started a few days before 2012 and continued steady when the new year came in. +It was around October 2011, during a short stay in San Francisco, that I came up with what would be [the foundation of Elixir's current version](https://github.com/josevalim/lego-lang) with the help of Yehuda Katz. Development of the new Elixir version started a few days before 2012 and continued steady when the new year came in. Around February of that year, feeling confident enough about the direction the language was moving (and initial benchmarks I had made at that point), I have pitched Elixir to [my company, Plataformatec](http://plataformatec.com.br/), and they have accepted to sponsor Elixir. With their help, Elixir developed even faster and that's what we are going to take a look next. @@ -27,9 +28,9 @@ Around February of that year, feeling confident enough about the direction the l One of the goals we have set was to have a good website and documentation before the next official release. With the help of the Plataformatec team, we created a logo for Elixir and put this website live. -At the same time, [we were working on pygments support](https://bitbucket.org/birkenfeld/pygments-main/pull-request/57/add-elixir-and-elixir-console-lexers), a [documentation generation tool](https://github.com/elixir-lang/ex_doc) and many others. Soon, Github was able to syntax highlight Elixir code and [our API documentation was online](http://elixir-lang.org). +At the same time, [we were working on pygments support](https://bitbucket.org/birkenfeld/pygments-main/pull-request/57/add-elixir-and-elixir-console-lexers), a [documentation generation tool](https://github.com/elixir-lang/ex_doc) and many others. Soon, GitHub was able to syntax highlight Elixir code and [our API documentation was online](/). -At the same time, people started to gather around #elixir-lang channel on irc.freenode.net and [play with Elixir](http://github.com/elixir-lang/mix), [start their](https://github.com/guedes/exdate) [own projects](https://github.com/yrashk/validatex) and [tutorials](https://github.com/alco/elixir/wiki/Erlang-Syntax:-A-Crash-Course). +At the same time, people started to gather around #elixir-lang channel on irc.freenode.net (now migrated to #elixir on irc.libera.chat) to [play with Elixir](https://github.com/elixir-lang/elixir/tree/main/lib/mix), [start their](https://github.com/guedes/exdate) [own projects](https://github.com/yrashk/validatex) and [tutorials](https://github.com/alco/elixir/wiki/Erlang-Syntax:-A-Crash-Course). Although the initial release was scheduled to April 2012, the feedback from such early developers forced us to review some design and syntax decisions and were extremely important to shape the language as it is today. @@ -37,12 +38,12 @@ With v0.5.0 finally out, we are committing to a stable syntax and a basic standa ## Looking forward -There are still many, many things to do! In the next months, we will continue working on growing our community, talks and other documentation material. A huge thanks to [Alexei Sholik](http://twitter.com/true_droid) who is moving this area forward. +There are still many, many things to do! In the next months, we will continue working on growing our community, talks and other documentation material. A huge thanks to [Alexei Sholik](https://twitter.com/true_droid) who is moving this area forward. -We will also work on better integration and documentation on building Erlang systems. Erlang ships with the [Open Telecom Platform](http://en.wikipedia.org/wiki/Open_Telecom_Platform) which provides many tools to build distributed applications. In v0.5.0, all these tools are already available but we want to make the build process even simpler. +We will also work on better integration and documentation on building Erlang systems. Erlang ships with the [Open Telecom Platform](https://en.wikipedia.org/wiki/Open_Telecom_Platform) which provides many tools to build distributed applications. In v0.5.0, all these tools are already available but we want to make the build process even simpler. -In parallel, we will improve our [documentation generation tool](https://github.com/elixir-lang/ex_doc) and [build tool](https://github.com/elixir-lang/mix) which will likely be merged into core when they are solid enough. +In parallel, we will improve our [documentation generation tool](https://github.com/elixir-lang/ex_doc) and [build tool](https://github.com/elixir-lang/elixir/tree/main/lib/mix) which will likely be merged into core when they are solid enough. Finally, we will continue improving the Standard Library. Although Elixir's goal is to rely on Erlang the most as possible, we also want to provide a small Standard Library which makes better use of Elixir semantics. For the next weeks, we will focus on improving the IO and File manipulation modules. New data types may also appear, for example, ranges come to my mind. -Check out our [home page](http://elixir-lang.org/) and the [getting started guide](http://elixir-lang.org/getting_started/1.html) for more information. Welcome aboard and grab a cup of Elixir, because you are certainly going to enjoy the ride! \ No newline at end of file +Check out our [home page](/) and the [getting started guide](https://hexdocs.pm/elixir/introduction.html) for more information. Welcome aboard and grab a cup of Elixir, because you are certainly going to enjoy the ride! diff --git a/_posts/2012-08-01-elixir-v0-6-0-released.markdown b/_posts/2012-08-01-elixir-v0-6-0-released.markdown index 3e52772bb..becafdf7c 100644 --- a/_posts/2012-08-01-elixir-v0-6-0-released.markdown +++ b/_posts/2012-08-01-elixir-v0-6-0-released.markdown @@ -1,28 +1,28 @@ --- layout: post title: Elixir v0.6.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: We have finally released Elixir v0.6.0! This release includes a build tool called Mix, support for Erlang typespecs, many improvements to IEx and improved IO, File and Macro support. --- -We have finally released [Elixir](http://elixir-lang.org/) v0.6.0! This release includes a build tool called Mix, support for Erlang typespecs, many improvements to IEx and improved IO, File and Macro support. +We have finally released [Elixir](/) v0.6.0! This release includes a build tool called Mix, support for Erlang typespecs, many improvements to IEx and improved IO, File and Macro support. ## What's new -When [we released version v0.5.0](http://elixir-lang.org/blog/2012/05/25/elixir-v0-5-0-released/), we have set three major goals for release v0.6.0: +When [we released version v0.5.0](/blog/2012/05/25/elixir-v0-5-0-released/), we have set three major goals for release v0.6.0: 1. Provide a build tool that makes it easy to create, compile and test Elixir projects; 2. Support [Erlang typespecs](http://www.erlang.org/doc/reference_manual/typespec.html); 3. Improve IO and File modules to be more robust and complete. -We have not only achieved those goals for this release, as we have added much more! A couple weeks ago, we have covered some of these unscheduled improvements, as improved Macro handling and Range support, which you can read more about in the ["What's new in Elixir #5" post](http://elixir-lang.org/blog/2012/07/05/what-s-new-in-elixir-5/). - Our interactive shell (IEx) also had many improvements, thanks to the Elixir developer community. We now have easy access to documentation, remote shells, autocomplete and much more. In order to show you a bit of what you can do in this release, we have prepared a short (~6 min) screencast: -

Elixir v0.6 quick tour - Mix and IEx from Plataformatec on Vimeo.

+ +Elixir v0.6 quick tour - Mix and IEx from Plataformatec on Vimeo. -That's it. Of course the documentation was also improved in the process, including two brand new getting started chapters on [Mix](/getting_started/mix/1.html) and [ExUnit](/getting_started/ex_unit/1.html). For the next months, we will continue improving Elixir (you can see some ideas floating around in the [issues tracker](github.com/elixir-lang/elixir/issues)) but we will start to focus on other tools and libraries for the community. +That's it. For the next months, we will continue improving Elixir (you can see some ideas floating around in the [issues tracker](https://github.com/elixir-lang/elixir/issues)) but we will start to focus on other tools and libraries for the community. -Thank you and don't forget to [give Elixir a try](/getting_started/1.html)! \ No newline at end of file +Thank you and don't forget to [give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2012-10-20-elixir-v0-7-0-released.markdown b/_posts/2012-10-20-elixir-v0-7-0-released.markdown index 40e6c29e3..d2f181612 100644 --- a/_posts/2012-10-20-elixir-v0-7-0-released.markdown +++ b/_posts/2012-10-20-elixir-v0-7-0-released.markdown @@ -1,13 +1,14 @@ --- layout: post title: Elixir v0.7.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.7.0 is released with many improvements! Read on for more information. --- -Elixir v0.7.0 was released with bug fixes and many improvements, like a `String` module to handle utf-8 binaries and support to environments and nested dependencies in Mix. +Elixir v0.7.0 was released with bug fixes and many improvements, like a `String` module to handle UTF-8 binaries and support to environments and nested dependencies in Mix. We have also taken important steps into normalizing our APIs. In Erlang, accesses to tuple and lists are one-based and binaries are zero-based, but in Elixir we have normalized all of them to rely on zero-based access. @@ -15,4 +16,4 @@ This release also includes some backwards incompatible changes, but the majority For more information, read out the [CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.7.0/CHANGELOG.md). -Thank you and don't forget to [give Elixir a try](/getting_started/1.html)! \ No newline at end of file +Thank you and don't forget to [give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2012-11-18-elixir-v0-7-1-released.markdown b/_posts/2012-11-18-elixir-v0-7-1-released.markdown index bfe465b6d..113bcd31f 100644 --- a/_posts/2012-11-18-elixir-v0-7-1-released.markdown +++ b/_posts/2012-11-18-elixir-v0-7-1-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.7.1 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.7.1 was released to celebrate the end of a two months journey traveling around Europe, United States and Brazil talking about Elixir. @@ -11,11 +12,11 @@ Elixir v0.7.1 was released this weekend to celebrate the end of a two months jou This is a minor release that contains a couple enhancements regarding UTF-8, [dialyzer](http://www.erlang.org/doc/man/dialyzer.html) support and bug fixes. -During this time traveling around, we have spoken at many conferences, as [Strange Loop](http://thestrangeloop.com/), [Øredev](http://oredev.org/), [QCon SP](http://qconsp.com/) and [Rupy](http://rupy.eu/) as well as at different companies. Developers from different backgrounds have shown interest in Elixir, [written about it](http://spin.atomicobject.com/2012/10/31/elixir-erlang-and-the-dining-philosophers/), joined us at #elixir-lang on freenode and contributed to the language. As of today, Elixir is powered by 51 different contributors! +During this time traveling around, we have spoken at many conferences, as [Strange Loop](http://thestrangeloop.com/), [Øredev](http://oredev.org/), [QCon SP](http://qconsp.com/) and [Rupy](http://rupy.eu/) as well as at different companies. Developers from different backgrounds have shown interest in Elixir, [written about it](http://spin.atomicobject.com/2012/10/31/elixir-erlang-and-the-dining-philosophers/), and contributed to the language. As of today, Elixir is powered by 51 different contributors! -In case you missed any of those conferences, [the talk I presented at Øredev is available and you can watch it now](http://vimeo.com/53221562). The slides are also available below. +In case you missed any of those conferences, [the talk I presented at Øredev is available and you can watch it now](https://vimeo.com/53221562). The slides are also available below. -If you want to hear more about Elixir at a conference or an event, please let us know. Thank you and don't forget to [give Elixir a try](/getting_started/1.html)! +If you want to hear more about Elixir at a conference or an event, please let us know. Thank you and don't forget to [give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2012-12-04-elixir-v0-7-2-released.markdown b/_posts/2012-12-04-elixir-v0-7-2-released.markdown index 27965bb1e..b4ec972c7 100644 --- a/_posts/2012-12-04-elixir-v0-7-2-released.markdown +++ b/_posts/2012-12-04-elixir-v0-7-2-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.7.2 released -author: Yurii Rashkovskii +authors: +- Yurii Rashkovskii category: Releases excerpt: Elixir v0.7.2 is released, new, improved type specifications syntax and many other improvements. @@ -44,10 +45,10 @@ We've also added a more compact and visual form of the `function` helper. Now, instead of `function(Enum, :all?, 2)` you can use `function(Enum.all?/2)`. We've also figured out how to achieve an up to 6x [performance increase](https://github.com/elixir-lang/elixir/blob/v0.7.2/lib/elixir/lib/kernel.ex#L1386-L1417) -under some circunstances when using records. +under some circumstances when using records. ...and [many other fixes & improvements](https://github.com/elixir-lang/elixir/blob/v0.7.2/CHANGELOG.md). Lastly, but not least importantly, I'd like to mention that we're very excited about how the community around Elixir is building up. Thank you all for being around and supporting us! -[Learn more about Elixir](/getting_started/1.html)! +[Learn more about Elixir](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2013-01-27-elixir-v0-8-0-released.markdown b/_posts/2013-01-27-elixir-v0-8-0-released.markdown index f7932babb..9f6236b10 100644 --- a/_posts/2013-01-27-elixir-v0-8-0-released.markdown +++ b/_posts/2013-01-27-elixir-v0-8-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.8.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: On the last 9th January, we celebrated two years since Elixir's first commit and to celebrate this occasion we have prepared a big release. Elixir v0.8 is out, with documentation, optimizations, bug fixes and shiny new features. Let's take a look at them! --- @@ -18,11 +19,11 @@ And applications can be started directly from the command line as well: elixir --app my_app -We have written a whole [guide chapter about creating OTP applications, supervisors and servers](/getting_started/mix/2.html). Give it a try! +We have written a whole [guide chapter about creating OTP applications, supervisors and servers](https://hexdocs.pm/elixir/supervisor-and-application.html). Give it a try! ## Improved Unicode support -Elixir favors the use of utf-8 binaries since its first release. In the latest releases, we took it up a notch by adding Unicode support, built upon the Unicode Standard 6.2.0. Elixir v0.8 takes this even further, adding more convenience functions and better support to named sequences: +Elixir favors the use of UTF-8 binaries since its first release. In the latest releases, we took it up a notch by adding Unicode support, built upon the Unicode Standard 6.2.0. Elixir v0.8 takes this even further, adding more convenience functions and better support to named sequences: ```elixir String.capitalize("fiN") #=> "Fin" @@ -30,7 +31,7 @@ String.capitalize("fiN") #=> "Fin" The example above contains a string with only two codepoints, [the codepoint fi](http://www.fileformat.info/info/unicode/char/FB01/index.htm) and [the codepoint n](http://www.fileformat.info/info/unicode/char/006E/index.htm). Look how Elixir properly capitalizes the string, returning a new string made of three codepoints (all ascii letters). -Learn more about [Unicode support with the String module](http://elixir-lang.org/docs/master/String.html). +Learn more about [Unicode support with the String module](https://hexdocs.pm/elixir/String.html). ## AST metadata @@ -70,15 +71,15 @@ When some code call the `unless` macro above, in previous Elixir versions, it wo Elixir v0.8 ensures that the `unless` macro above will expand to the same `if` macro available when quoted, guaranteeing different libraries can integrate easily without imposing hidden requirements. -You can read more about [macros in the getting started guide](http://elixir-lang.org/getting_started/5.html) or [go deep into the quote macro docs](http://elixir-lang.org/docs/master/Kernel.SpecialForms.html#quote/2). +You can read more about [macros in the getting started guide](https://hexdocs.pm/elixir/case-cond-and-if.html) or [go deep into the quote macro docs](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#quote/2). ## A new way to manipulate pathnames -Elixir v0.8 contains a bit of house cleaning too. We have created [the Path module](http://elixir-lang.org/docs/master/Path.html) to accommodate functions used to manipulate filesystem paths and have also added functions like [`System.tmp_dir` and `System.user_home`](http://elixir-lang.org/docs/master/System.html) which are meant to work accross different operating systems and are very handy when scripting. +Elixir v0.8 contains a bit of house cleaning too. We have created [the Path module](https://hexdocs.pm/elixir/Path.html) to accommodate functions used to manipulate filesystem paths and have also added functions like [`System.tmp_dir` and `System.user_home`](https://hexdocs.pm/elixir/System.html) which are meant to work across different operating systems and are very handy when scripting. ## The new HashDict -For last but not least, Elixir ships with a [new HashDict implementation](https://github.com/elixir-lang/elixir/blob/master/lib/elixir/lib/hash_dict.ex). In Erlang, there are different key-value store implementations and often you need to pick which one is the best for you based on the average size of the dictionary. Generally speaking, [orddicts](http://www.erlang.org/doc/man/orddict.html) are efficient and fast when you want to hold a handful of items, otherwise you should consider [gb_trees](http://www.erlang.org/doc/man/gb_trees.html) unless you want to hold thousands of items, when then [dict](http://www.erlang.org/doc/man/dict.html) becomes your best option. The fact those implementations do not provide the same API, makes it harder to change your code when you realize another implementation would be better fit. +For last but not least, Elixir ships with a [new HashDict implementation](https://github.com/elixir-lang/elixir/blob/main/lib/elixir/lib/hash_dict.ex). In Erlang, there are different key-value store implementations and often you need to pick which one is the best for you based on the average size of the dictionary. Generally speaking, [orddicts](http://www.erlang.org/doc/man/orddict.html) are efficient and fast when you want to hold a handful of items, otherwise you should consider [gb_trees](http://www.erlang.org/doc/man/gb_trees.html) unless you want to hold thousands of items, when then [dict](http://www.erlang.org/doc/man/dict.html) becomes your best option. The fact those implementations do not provide the same API, makes it harder to change your code when you realize another implementation would be better fit. For Elixir, we decided to have a single dictionary implementation that would scale as needed. It would start as a compact representation for a handful of items and expand and rehash accordingly as new items are added or removed, providing fast access and modification times on all ranges. We are glad to say our goals were reached and a new `HashDict` implementation ships with Elixir v0.8. @@ -100,4 +101,4 @@ We continue actively working on Elixir and this release is the [result of our ef Also, we previously announced Elixir is going to be released frequently, every 2 to 4 weeks. We have made a small detour to get v0.8.0 out of the door, but we are back to our regular schedule as of today! -[Celebrate with us and give Elixir a try](/getting_started/1.html)! +[Celebrate with us and give Elixir a try](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2013-04-19-google-summer-of-code-2013.markdown b/_posts/2013-04-19-google-summer-of-code-2013.markdown index b025579f6..d964d7c1b 100644 --- a/_posts/2013-04-19-google-summer-of-code-2013.markdown +++ b/_posts/2013-04-19-google-summer-of-code-2013.markdown @@ -1,7 +1,8 @@ --- layout: post title: Google Summer of Code 2013 -author: José Valim +authors: +- José Valim category: Announcements excerpt: Elixir is taking part in Google Summer of Code 2013! Are you a student? Join us! --- diff --git a/_posts/2013-04-29-elixir-v0-8-2-released.markdown b/_posts/2013-04-29-elixir-v0-8-2-released.markdown index 762f7004a..fea64d566 100644 --- a/_posts/2013-04-29-elixir-v0-8-2-released.markdown +++ b/_posts/2013-04-29-elixir-v0-8-2-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.8.2 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.8.2 is released with bug fixes, better Erlang R16 support and doctests. --- @@ -44,6 +45,6 @@ defmodule MathTest do end ``` -You can learn more about [doctests on our documentation page](http://elixir-lang.org/docs/stable/ex_unit/ExUnit.DocTest.html) and get more information about our latest release [on the CHANGELOG](https://github.com/elixir-lang/elixir/blob/ed27611f48ba150404c95fe15f1d6058a4287330/CHANGELOG.md). +You can learn more about [doctests on our documentation page](https://hexdocs.pm/ex_unit/ExUnit.DocTest.html) and get more information about our latest release [on the CHANGELOG](https://github.com/elixir-lang/elixir/blob/ed27611f48ba150404c95fe15f1d6058a4287330/CHANGELOG.md). -If you are new to Elixir, [it's easy to get started with](/getting_started/1.html)! +If you are new to Elixir, [it's easy to get started with](https://hexdocs.pm/elixir/introduction.html)! diff --git a/_posts/2013-05-02-elixir-on-xen.markdown b/_posts/2013-05-02-elixir-on-xen.markdown index 0bbd9829a..9f1de4c2b 100644 --- a/_posts/2013-05-02-elixir-on-xen.markdown +++ b/_posts/2013-05-02-elixir-on-xen.markdown @@ -1,24 +1,25 @@ --- layout: post title: Elixir on Xen -author: José Valim +authors: +- José Valim category: Announcements excerpt: The Erlang on Xen team has added support for Elixir and we will tell you how you can use it! --- -Elixir uses Erlang underneath, all the way down. Thanks to this, an Elixir project can run on the recently revealed “OS-less” Erlang VM called LING VM. LING VM is the core technology of [Erlang on Xen](http://erlangonxen.org). +Elixir uses Erlang underneath, all the way down. Thanks to this, an Elixir project can run on the recently revealed "OS-less" Erlang VM called LING VM. LING VM is the core technology of [Erlang on Xen](http://erlangonxen.org). ## Why Xen? [Xen](https://en.wikipedia.org/wiki/Xen) is an open-source baremetal hypervisor that allows many operating systems to run on the same hardware. Xen is frequently used for server virtualization, Infrastructure as a Service (IaaS) and security applications. -Elixir on Xen runs on top of the Xen Hypervisor (via the LING VM) but with no traditional OS underneath it, taking away numerous administrative, scalability, and performance issues. This limits options of a malicious attacker, making it an excellent choice for high-security applications, and reduces startup latency, allowing developers to spawn new VMs in less than 100 miliseconds. +Elixir on Xen runs on top of the Xen Hypervisor (via the LING VM) but with no traditional OS underneath it, taking away numerous administrative, scalability, and performance issues. This limits options of a malicious attacker, making it an excellent choice for high-security applications, and reduces startup latency, allowing developers to spawn new VMs in less than 100 milliseconds. You can learn more about Xen and the LING VM on the [Erlang on Xen website](http://erlangonxen.org). ## Getting started -In order to run Elixir on the LING VM, you need to produce a Xen image of your Elixir project. This can be done with the help of the [lingex project](http://github.com/maximk/lingex), created by the LING VM team. +In order to run Elixir on the LING VM, you need to produce a Xen image of your Elixir project. This can be done with the help of the [lingex project](https://github.com/maximk/lingex), created by the LING VM team. Producing an Elixir image using the free Erlang on Xen Build Service requires just a few steps: diff --git a/_posts/2013-05-23-elixir-v0-9-0-released.markdown b/_posts/2013-05-23-elixir-v0-9-0-released.markdown index 168928bb9..edee304e1 100644 --- a/_posts/2013-05-23-elixir-v0-9-0-released.markdown +++ b/_posts/2013-05-23-elixir-v0-9-0-released.markdown @@ -1,12 +1,13 @@ --- layout: post title: Elixir v0.9.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.9.0 is released with support for reducers, umbrella projects, faster compilation times and dropped support for R15 and earlier OTP versions. --- -While [Programming Elixir](http://pragprog.com/book/elixir/programming-elixir) was being announced, we have been working on Elixir v0.9.0 which is finally out. This release contains new features, important performance optimizations and bug fixes. +While [Programming Elixir](https://pragprog.com/book/elixir/programming-elixir) was being announced, we have been working on Elixir v0.9.0 which is finally out. This release contains new features, important performance optimizations and bug fixes. Elixir v0.9.0 also removes support for Erlang R15 and earlier versions. In case you still need to run Elixir software on R15, we have also released Elixir v0.8.3, which contains many of the enhancements in v0.9.0. Check the [CHANGELOG for more details for both releases](https://github.com/elixir-lang/elixir/blob/v0.9.0/CHANGELOG.md). @@ -36,7 +37,9 @@ As a project grows, it is recommended to break it apart into smaller, isolated a Elixir v0.9.0 now supports umbrella projects which can work with many applications at the same time. You can create a new umbrella project with: - $ mix new my_project --umbrella +```bash +$ mix new my_project --umbrella +``` The generated project will have the following structure: @@ -53,11 +56,11 @@ A special thanks to [Eric Meadows-Jonsson](https://github.com/ericmj) for implem Elixir v0.9.0 changes its main abstraction for enumeration from iterators to reducers. Before Elixir v0.9.0, when you invoked: ```elixir -Enum.map([1,2,3], fn(x) -> x * x end) +Enum.map([1, 2, 3], fn(x) -> x * x end) #=> [1, 4, 9] ``` -It asked the `Enum.Iterator` protocol for instructions on how to iterate the list `[1,2,3]`. This iteration happened by retrieving each item in the list, one by one, until there were no items left. +It asked the `Enum.Iterator` protocol for instructions on how to iterate the list `[1, 2, 3]`. This iteration happened by retrieving each item in the list, one by one, until there were no items left. This approach posed many problems: @@ -75,8 +78,8 @@ defimpl Enumerable, for: List do do_reduce(list, acc, fun) end - defp do_reduce([h|t], acc, fun) do - do_reduce(t, fun.(h, acc), fun) + defp do_reduce([head | tail], acc, fun) do + do_reduce(tail, fun.(head, acc), fun) end defp do_reduce([], acc, fun) do @@ -89,7 +92,7 @@ The implementation above works as a simple `reduce` function (also called `fold` ```elixir # Sum all elements in a list -Enumerable.reduce([1,2,3], 0, fn(x, acc) -> x + acc end) +Enumerable.reduce([1, 2, 3], 0, fn(x, acc) -> x + acc end) #=> 6 ``` @@ -99,7 +102,7 @@ The `Enum.map/2` we have used above is now implemented in terms of this reducing defmodule Enum do def map(collection, fun) do Enumerable.reduce(collection, [], fn(x, acc) -> - [fun.(x, acc)|acc] + [fun.(x, acc) | acc] end) |> reverse end end @@ -121,11 +124,11 @@ A special thanks to [Eric Meadows-Jonsson](https://github.com/ericmj) for implem We have also many other smaller improvements: * Our CLI now supports `--hidden` and `--cookie` flags which are useful for distributed modes; -* Our test framework, ExUnit, is now able to capture all the communication that happens with a registed IO device, like `:stdio` and `:stderr`, via [`ExUnit.CaptureIO`](http://elixir-lang.org/docs/master/ExUnit.CaptureIO.html). This is very useful for testing how your software reacts to some inputs and what it prints to the terminal; +* Our test framework, ExUnit, is now able to capture all the communication that happens with a registered IO device, like `:stdio` and `:stderr`, via [`ExUnit.CaptureIO`](https://hexdocs.pm/ex_unit/ExUnit.CaptureIO.html). This is very useful for testing how your software reacts to some inputs and what it prints to the terminal; * `IEx` now allows files to be imported into the shell with `import_file` and also loads `~/.iex` on startup for custom configuration; * The `String`, `Enum` and `Dict` modules got more convenience functions that goes from checking unicode character validity to taking values out of a dictionary; * And many, many more! A huge thank you to our community for sending bug reports, providing bug fixes and contributing all those amazing features. And when are **you** joining us? :) -Give Elixir a try! You can start with our [getting started guide](http://elixir-lang.org/getting_started/1.html), or [check this 30 minute video from PragProg](http://www.youtube.com/watch?v=a-off4Vznjs&feature=youtu.be) or buy the beta version of [Programming Elixir](http://pragprog.com/book/elixir/programming-elixir). +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or [check this 30 minute video from PragProg](https://www.youtube.com/watch?v=a-off4Vznjs&feature=youtu.be) or buy the beta version of [Programming Elixir](https://pragprog.com/book/elixir/programming-elixir). diff --git a/_posts/2013-07-13-elixir-v0-10-0-released.markdown b/_posts/2013-07-13-elixir-v0-10-0-released.markdown index 14716c4a1..c00858ea8 100644 --- a/_posts/2013-07-13-elixir-v0-10-0-released.markdown +++ b/_posts/2013-07-13-elixir-v0-10-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.10.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.10.0 is out with support for streams, sets, pretty printing and many improvements for Mix and ExUnit. --- @@ -13,17 +14,17 @@ Elixir v0.10.0 is released with support for streams, sets and many improvements The default mechanism for working with collections in Elixir is the `Enum` module. With it, you can map over ranges, lists, sets, dictionaries and any other structure as long as it implements the `Enumerable` protocol: ```elixir -Enum.map([1,2,3], fn(x) -> x * 2 end) -#=> [2,4,6] +Enum.map([1, 2, 3], fn(x) -> x * 2 end) +#=> [2, 4, 6] ``` The `Enum` module performs eager evaluation. Consider the following example: ```elixir -[1,2,3] +[1, 2, 3] |> Enum.take_while(fn(x) -> x < 3 end) |> Enum.map(fn(x) -> x * 2 end) -#=> [2,4] +#=> [2, 4] ``` In the example above, we enumerate the items in list once, taking all elements that are less than 3, and then we enumerate the remaining elements again, multiplying them by two. In order to retrieve the final result, we have created one intermediate list. As we add more operations, more intermediate lists will be generated. @@ -31,7 +32,7 @@ In the example above, we enumerate the items in list once, taking all elements t This approach is simple and efficient for the majority of the cases but, when working with large collections, we can generate many, possibly large, intermediate lists affecting performance. That's one of the problems Streams solve. Let's rewrite the example above using Streams: ```elixir -[1,2,3] +[1, 2, 3] |> Stream.take_while(fn(x) -> x < 3 end) |> Stream.map(fn(x) -> x * 2 end) #=> #Stream.Lazy<...> @@ -54,11 +55,11 @@ Stream.repeatedly(fn -> :random.uniform end) |> Enum.take(3) #=> [0.4435846174457203, 0.7230402056221108, 0.94581636451987] ``` -`Stream.repeatedly/1` returns an infinite stream but that's ok we just need its first three elements. You can learn more about [stream and related functions in `Stream` module documentation](http://elixir-lang.org/docs/stable/elixir/Stream.html). +`Stream.repeatedly/1` returns an infinite stream but that's ok we just need its first three elements. You can learn more about [stream and related functions in `Stream` module documentation](https://hexdocs.pm/elixir/Stream.html). ## Sets -This release also adds [the Sets API](http://elixir-lang.org/docs/stable/elixir/Set.html) to Elixir and a HashSet implementation. The HashSet implementation follows [the same design goals as the HashDict implementation](http://elixir-lang.org/blog/2013/01/27/elixir-v0-8-0-released/) released at the beginning of this year, starting with a compact representation and expanding and contracting as needed. +This release also adds [the Sets API](https://hexdocs.pm/elixir/Set.html) to Elixir and a HashSet implementation. The HashSet implementation follows [the same design goals as the HashDict implementation](/blog/2013/01/27/elixir-v0-8-0-released/) released at the beginning of this year, starting with a compact representation and expanding and contracting as needed. This feature was a contribution from [Joseph Wilk](https://github.com/josephwilk) and he talks about its implementation and provides some benchmarks [on his blog](http://blog.josephwilk.net/elixir/sets-in-elixir.html). @@ -66,7 +67,7 @@ This feature was a contribution from [Joseph Wilk](https://github.com/josephwilk Another addition to this release is pretty printing. The pretty printing started as an implementation of the [Wadler paper](http://homepages.inf.ed.ac.uk/wadler/papers/prettier/prettier.pdf) by [Jonns Mostovoys](https://github.com/manpages) which was then improved by [Gustavo Brunoro](https://github.com/brunoro) under his Google Summer of Code project as described in [Lindig's _Strictly Prettier_ paper](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.34.2200). -As soon as you upgrade to Elixir v0.10.0 and start IEx, you will get pretty printing for all data structures provided by Elixir. We have also added documentation to the `Inspect` module about [adding pretty printing to your own structures](http://elixir-lang.org/docs/stable/elixir/Inspect.html) as well as [using the document algebra for any other kind of formatting](http://elixir-lang.org/docs/stable/elixir/Inspect.Algebra.html). +As soon as you upgrade to Elixir v0.10.0 and start IEx, you will get pretty printing for all data structures provided by Elixir. We have also added documentation to the `Inspect` module about [adding pretty printing to your own structures](https://hexdocs.pm/elixir/Inspect.html) as well as [using the document algebra for any other kind of formatting](https://hexdocs.pm/elixir/Inspect.Algebra.html). ## Other improvements @@ -78,8 +79,8 @@ Other notable improvements are: * We are also working hard on Windows support, improving its command-line tools and working towards a green test suite, thanks to [Tom Jansens](https://github.com/tojans); -* Meta-programming in Elixir was also improved by the addition of the `binding/0` and `binding/1` macros plus the additions of `Macro.expand_once/2` and `Macro.expand_all/2` to the [`Macro` module](http://elixir-lang.org/docs/stable/elixir/Macro.html); +* Meta-programming in Elixir was also improved by the addition of the `binding/0` and `binding/1` macros plus the additions of `Macro.expand_once/2` and `Macro.expand_all/2` to the [`Macro` module](https://hexdocs.pm/elixir/Macro.html); There are also improvements to typespecs, error messages, many bug fixes and some backwards incompatible changes. We have posted a detailed [upgrade instructions on the mailing list](https://groups.google.com/forum/?fromgroups#!topic/elixir-lang-talk/ksrefrgK1eY). For a general overview, [check out the CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.10.0/CHANGELOG.md). -Give Elixir a try! You can start with our [getting started guide](http://elixir-lang.org/getting_started/1.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. diff --git a/_posts/2013-08-08-elixir-design-goals.markdown b/_posts/2013-08-08-elixir-design-goals.markdown index 55e86d5e9..5959dab92 100644 --- a/_posts/2013-08-08-elixir-design-goals.markdown +++ b/_posts/2013-08-08-elixir-design-goals.markdown @@ -1,12 +1,13 @@ --- layout: post title: Elixir Design Goals -author: José Valim +authors: +- José Valim category: Internals excerpt: Highlight of Elixir design goals. --- -During the last year, we have spoken at many conferences spreading the word about Elixir. We [usually started with introducing the Erlang VM](http://vimeo.com/53221562), then went on to talk about Elixir goals, saving some time at the end to do a live demo, showing some goodies like exchanging information between remote nodes and even hot code swapping. +During the last year, we have spoken at many conferences spreading the word about Elixir. We [usually started with introducing the Erlang VM](https://vimeo.com/53221562), then went on to talk about Elixir goals, saving some time at the end to do a live demo, showing some goodies like exchanging information between remote nodes and even hot code swapping. This post is a summary of those talks, focusing on the language goals: compatibility, productivity and extensibility. @@ -63,7 +64,7 @@ ExUnit.start defmodule MathTest do use ExUnit.Case, async: true - + test "adding two numbers" do assert 1 + 2 == 4 end @@ -125,7 +126,7 @@ defmodule Hello, do: ( ) ``` -And finally we added `do/end` as convenience for the common `do: (...)` construct: +And finally we added `do`-`end` as convenience for the common `do: (...)` construct: ```elixir defmodule Hello do @@ -136,7 +137,7 @@ defmodule Hello do end ``` -Given my previous background in Ruby, it is natural that some of the constructs added were borrowed from Ruby. However, those additions were a by-product, never a language goal. +Given my previous background in Ruby, it is natural that some of the constructs added were borrowed from Ruby. However, those additions were a by-product, and not a language goal. Many language constructs are also inspired by their Erlang counter-parts, like some of the control-flow macros, operators and containers. Notice how some Elixir code: @@ -145,7 +146,7 @@ Many language constructs are also inspired by their Erlang counter-parts, like s tuple = { 1, 2, 3 } # Adding two lists -[1,2,3] ++ [4,5,6] +[1, 2, 3] ++ [4, 5, 6] # Case case expr do @@ -161,7 +162,7 @@ maps to Erlang: Tuple = { 1, 2, 3 }. % Adding two lists -[1,2,3] ++ [4,5,6]. +[1, 2, 3] ++ [4, 5, 6]. % Case case Expr of @@ -189,27 +190,27 @@ And much more. Most of the features above provide their own extensibility mechanisms, too. For example, take the `Enum` module. The `Enum` module allow us to enumerate the built-in ranges, lists, sets, etc: ```elixir -list = [1,2,3] +list = [1, 2, 3] Enum.map list, fn(x) -> x * 2 end -#=> [2,4,6] +#=> [2, 4, 6] range = 1..3 Enum.map range, fn(x) -> x * 2 end -#=> [2,4,6] +#=> [2, 4, 6] -set = HashSet.new [1,2,3] +set = HashSet.new [1, 2, 3] Enum.map set, fn(x) -> x * 2 end -#=> [2,4,6] +#=> [2, 4, 6] ``` -Not only that, any developer can **extend** the `Enum` module to work with any data type as long as the data type implements [the `Enumerable` protocol](http://elixir-lang.org/docs/stable/elixir/Enumerable.html) (protocols in Elixir are based on Clojure's protocol). This is extremely convenient because the developer needs to know only the `Enum` API for enumeration, instead of memorizing specific APIs for sets, lists, dicts, etc. +Not only that, any developer can **extend** the `Enum` module to work with any data type as long as the data type implements [the `Enumerable` protocol](https://hexdocs.pm/elixir/Enumerable.html) (protocols in Elixir are based on Clojure's protocol). This is extremely convenient because the developer needs to know only the `Enum` API for enumeration, instead of memorizing specific APIs for sets, lists, dicts, etc. -There are many other protocols exposed by the language, like [the `Inspect` protocol](http://elixir-lang.org/docs/stable/elixir/Inspect.html) for pretty printing data structures and [the `Access` protocol](http://elixir-lang.org/docs/stable/elixir/Access.html) for accessing key-value data by key. By being extensible, Elixir ensures developers can work **with** the language, instead of **against** the language. +There are many other protocols exposed by the language, like [the `Inspect` protocol](https://hexdocs.pm/elixir/Inspect.html) for pretty printing data structures and [the `Access` protocol](https://hexdocs.pm/elixir/Access.html) for accessing key-value data by key. By being extensible, Elixir ensures developers can work **with** the language, instead of **against** the language. ## Summing up -The goal of this post was to sumarize the language goals: compatibility, productivity and extensibility. By being compatibile with the Erlang VM, we are providing developers another toolset for building concurrent, distributed and fault-tolerant systems. +The goal of this post was to sumarize the language goals: compatibility, productivity and extensibility. By being compatible with the Erlang VM, we are providing developers another toolset for building concurrent, distributed and fault-tolerant systems. We also hope to have clarified what Elixir brings to the Erlang VM, in particular, meta-programming through macros, polymorphic constructs for extensibility and a data-focused standard library with extensible and consistent APIs for diverse types, including strict and lazy enumeration, unicode handling, a test framework and more. -Give Elixir a try! You can start with our [getting started guide](http://elixir-lang.org/getting_started/1.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. diff --git a/_posts/2013-11-05-elixir-v0-11-0-released.markdown b/_posts/2013-11-05-elixir-v0-11-0-released.markdown index 6a31e2702..1a668262c 100644 --- a/_posts/2013-11-05-elixir-v0-11-0-released.markdown +++ b/_posts/2013-11-05-elixir-v0-11-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.11.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.11.0 is out and it focus on improving and optimizing the patterns often used by the community. --- @@ -22,11 +23,11 @@ In this new release, IEx also supports a very simple debugging mechanism called ![IEx pry example](/images/contents/iex-pry.png) -In Elixir, your code runs in many processes that talk to each other and the Elixir shell is no different. `IEx.pry` allows another process to take over the shell, allowing the developer to inspect the binding and halt the execution of the process being "pried" (i.e. the one that invoked `IEx.pry`). We called this feature `pry` as a gentle reminder that you can only inspect existing information, you cannot change the binding over a pried process. For more information, check the docs for [`IEx.pry/1`](http://elixir-lang.org/docs/stable/iex/IEx.html#pry/1). +In Elixir, your code runs in many processes that talk to each other and the Elixir shell is no different. `IEx.pry` allows another process to take over the shell, allowing the developer to inspect the binding and halt the execution of the process being "pried" (i.e. the one that invoked `IEx.pry`). We called this feature `pry` as a gentle reminder that you can only inspect existing information, you cannot change the binding over a pried process. For more information, check the docs for [`IEx.pry/1`](https://hexdocs.pm/iex/IEx.html#pry/1). ## ExUnit -[In the previous release](http://elixir-lang.org/blog/2013/07/13/elixir-v0-10-0-released/), we introduced great changes to ExUnit, like the support for the `--trace` option. This time we continued pushing improvements, like adding profiling to test cases (times can be seen with the `--trace` option), paving the way for other features like emitting warnings for test cases that are too slow. +[In the previous release](/blog/2013/07/13/elixir-v0-10-0-released/), we introduced great changes to ExUnit, like the support for the `--trace` option. This time we continued pushing improvements, like adding profiling to test cases (times can be seen with the `--trace` option), paving the way for other features like emitting warnings for test cases that are too slow. Another simple but significant change in ExUnit was the change in the default formatter to print changes as they come, instead of waiting until the suite is done running: @@ -36,7 +37,7 @@ This change allows developer to get faster feedback from their test suites. ## Mix -Since the early days, Elixir took ahold of the compilation process in order to provide a seamless compilation experience. [Elixir's ParallelCompiler](http://elixir-lang.org/blog/2012/04/24/a-peek-inside-elixir-s-parallel-compiler/) was introduced even before the first official release, allowing developers to harness all the cores in their computer to compile Elixir code. However, once the first release came out, every time you changed any file, the whole project had to be recompiled. +Since the early days, Elixir took ahold of the compilation process in order to provide a seamless compilation experience. [Elixir's ParallelCompiler](/blog/2012/04/24/a-peek-inside-elixir-s-parallel-compiler/) was introduced even before the first official release, allowing developers to harness all the cores in their computer to compile Elixir code. However, once the first release came out, every time you changed any file, the whole project had to be recompiled. In the past releases we have improved this process to only compile files that changed and their dependencies. For v0.11.0, we have improved this process to be faster and less conservative than the previous version. @@ -59,14 +60,14 @@ fun.(1..3) #=> true ``` -You can learn more about the [new capture operator in our docs](http://elixir-lang.org/docs/stable/elixir/Kernel.SpecialForms.html#&/1). +You can learn more about the [new capture operator in our docs](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#&/1). -We have also pushed improvements to [the String module](http://elixir-lang.org/docs/stable/elixir/String.html), including new APIs. In particular, in order to know that `String.length("josé")` has length 4 (even though it takes 5 bytes to be represented in UTF-8), we need to use some algorithms defined by the Unicode Standard. These have been implemented as specified in the [extended grapheme cluster algorithm, defined in the version 6.3.0 of the Unicode Standard](http://www.unicode.org/reports/tr29/). +We have also pushed improvements to [the String module](https://hexdocs.pm/elixir/String.html), including new APIs. In particular, in order to know that `String.length("josé")` has length 4 (even though it takes 5 bytes to be represented in UTF-8), we need to use some algorithms defined by the Unicode Standard. These have been implemented as specified in the [extended grapheme cluster algorithm, defined in the version 6.3.0 of the Unicode Standard](http://www.unicode.org/reports/tr29/). In the optimization front, we have pushed the first iteration of a [feature called Protocol consolidation](https://groups.google.com/forum/#!topic/elixir-lang-core/RoXAUtoyjk4), which speeds up the polymorphic dispatch done by protocols, sometimes reducing the dispatching time to 10% of the original time. We will continue working in upcoming releases to integrate protocol consolidation as a regular part of the developer workflow. And finally, a minor but frequently asked feature has finally arrived into Elixir: variables follow the same rules as other identifiers in the language, which means developers can now name their variables `is_atom?`. For a general overview, [check out the CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.11.0/CHANGELOG.md). -Give Elixir a try! You can start with our [getting started guide](http://elixir-lang.org/getting_started/1.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. **PS:** We have just released v0.11.1 which addresses a regression in Mix and improves the dependencies update process. diff --git a/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown b/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown index 717e04aeb..b29807cb4 100644 --- a/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown +++ b/_posts/2013-12-11-elixir-s-new-continuable-enumerators.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir's new continuable enumerators -author: Peter Minten +authors: +- Peter Minten category: Internals excerpt: In 0.12.0 Elixir's enumerators have gained the ability to suspend value production and to terminate early. @@ -89,7 +90,7 @@ the basis of a zip function. Without interleaving you cannot implement The underlying problem, in both cases, is that the producer is fully in control. The producer simply pushes out as many elements to the consumer as it wants and -then says "I'm done". There's no way aside from `throw`/`raise` for a consumer +then says "I'm done". There's no way aside from `throw/raise` for a consumer to tell a producer "stop producing". There is definitely no way to tell a producer "stop for now but be prepared to continue where you left off later". @@ -112,7 +113,7 @@ same as in the old system. A consumer may return `:halt` to have the producer terminate earlier than it normally would. The real magic is in `:suspend` though. It tells a producer to return the -accumulator and a continuation function. +accumulator and a continuation function. ```elixir { :suspended, n_, cont } = Enumerable.reduce(1..5, { :cont, 0 }, fn x, n -> @@ -152,7 +153,7 @@ function. ```elixir defmodule Interleave do def interleave(a, b) do - step = fn x, acc -> { :suspend, [x|acc] } end + step = fn x, acc -> { :suspend, [x | acc] } end af = &Enumerable.reduce(a, &1, step) bf = &Enumerable.reduce(b, &1, step) do_interleave(af, bf, []) |> :lists.reverse() @@ -186,7 +187,7 @@ defmodule Interleave do end end -Interleave.interleave([1,2], [:a, :b, :c, :d]) +Interleave.interleave([1, 2], [:a, :b, :c, :d]) #=> [1, :a, 2, :b, :c, :d] ``` diff --git a/_posts/2013-12-15-elixir-v0-12-0-released.markdown b/_posts/2013-12-15-elixir-v0-12-0-released.markdown index 200a4a5cf..fe8a349e3 100644 --- a/_posts/2013-12-15-elixir-v0-12-0-released.markdown +++ b/_posts/2013-12-15-elixir-v0-12-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.12.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: Elixir v0.12.0 is out with improved enumerables, build patterns and welcoming a new member to our team --- @@ -10,7 +11,7 @@ Elixir v0.12.0 has been released with improved enumerables, build patterns and w ## Enumerables -In previous versions, the Enumerable protocol was based on reduce/fold, and while it is very efficient for operations like `map`, `reduce` and `filter`, it was sub-optimal for operations that need to halt, like `take` and `take_while`, and it made it impossible for operations like `zip` to be implemented. +In previous versions, the Enumerable protocol was based on reduce/fold, and while it is very efficient for operations like `map`, `reduce` and `filter`, it was sub-optimal for operations that need to halt, like `take` and `take_while`, and it made it impossible for operations like `zip` to be implemented. In v0.12.0, Elixir's Enumerable protocol has been extended to allow suspension and halting mechanisms, making operations like `take` simpler and operations that require interleaving, like `zip`, possible. @@ -18,7 +19,7 @@ Although most users don't need to concern with the implementation of the Enumera ## Mix -The tool that received most improvements in this release was Mix. The biggest change is that Mix no longer compiles projects in place but to the `_build` directory. For example, take the [Ecto project](https://github.com/elixir-lang/ecto) that [depends on `postgrex` and `poolboy`](https://github.com/elixir-lang/ecto/blob/master/mix.exs#L24-L25). When compiled, all the artifacts will be placed in the `_build` directory like this: +The tool that received most improvements in this release was Mix. The biggest change is that Mix no longer compiles projects in place but to the `_build` directory. For example, take the [Ecto project](https://github.com/elixir-ecto/ecto) that [depends on `postgrex` and `poolboy`](https://github.com/elixir-ecto/ecto/blob/v0.1.0/mix.exs#L25-L30). When compiled, all the artifacts will be placed in the `_build` directory like this: ``` _build @@ -43,10 +44,10 @@ Mix has also added support to optional dependencies and improved common patterns With this release, we also want to welcome [Eric MJ](https://github.com/ericmj) to the Elixir Team. He has done fantastic work on Elixir, helping us maintain the codebase and working on many of the important features from previous releases and now many more to come. -Eric is also maintainer of both [Ecto](https://github.com/elixir-lang/ecto) and [Postgrex](https://github.com/ericmj/postgrex) projects. Which are proving to be very useful to the Elixir community too! +Eric is also maintainer of both [Ecto](https://github.com/elixir-ecto/ecto) and [Postgrex](https://github.com/elixir-ecto/postgrex) projects. Which are proving to be very useful to the Elixir community too! ## Tidying up There were other small changes, like additions to the `Float` module and improvements the to the typespec syntax. To see the full list, please [see the CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.12.0/CHANGELOG.md). -Give Elixir a try! You can start with our [getting started guide](http://elixir-lang.org/getting_started/1.html), or check out our sidebar for other learning resources. +Give Elixir a try! You can start with our [getting started guide](https://hexdocs.pm/elixir/introduction.html), or check out our sidebar for other learning resources. diff --git a/_posts/2014-04-21-elixir-v0-13-0-released.markdown b/_posts/2014-04-21-elixir-v0-13-0-released.markdown index 9e2e59baa..995e4d68c 100644 --- a/_posts/2014-04-21-elixir-v0-13-0-released.markdown +++ b/_posts/2014-04-21-elixir-v0-13-0-released.markdown @@ -1,14 +1,15 @@ --- layout: post title: Elixir v0.13.0 released, hex.pm and ElixirConf announced -author: José Valim +authors: +- José Valim category: Releases -excerpt: "Elixir v0.13.0 comes with substantial improvements to the language: maps, structs, comprehensiona and more. It also marks the announcement of the hex.pm package manager and the announcment of ElixirConf!" +excerpt: "Elixir v0.13.0 comes with substantial improvements to the language: maps, structs, comprehensiona and more. It also marks the announcement of the hex.pm package manager and the announcement of ElixirConf!" --- Hello folks! -Elixir v0.13.0 has been released. It contains changes that will effectively shape how developers will write Elixir code from now on, making it an important milestone towards v1.0! On this post we are going to cover some of those changes, the road to Elixir v1.0, as well as the announcement of [hex.pm](http://hex.pm). +Elixir v0.13.0 has been released. It contains changes that will effectively shape how developers will write Elixir code from now on, making it an important milestone towards v1.0! On this post we are going to cover some of those changes, the road to Elixir v1.0, as well as the announcement of [hex.pm](https://hex.pm). Before we go into the changes, let's briefly talk about ElixirConf! @@ -28,21 +29,21 @@ In a nutshell, here is what new: * Elixir v0.13 also provides structs, an alternative to Elixir records. Structs are more flexible than records, provide faster polymorphic operations, and still provide the same compile-time guarantees many came to love in records; -* The [Getting Started guide](http://elixir-lang.org/getting_started/1.html) was rewritten from scratch. The previous guide was comprised of 7 chapters and was about to become 2 years old. The new guide features 20 chapters, it explores the new maps and structs (which are part of this release), and it goes deeper into topics like IO and File handling. It also includes an extra guide, still in development, about [Meta-Programming in Elixir](http://elixir-lang.org/getting_started/meta/1.html); +* The [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) was rewritten from scratch. The previous guide was comprised of 7 chapters and was about to become 2 years old. The new guide features 20 chapters, it explores the new maps and structs (which are part of this release), and it goes deeper into topics like IO and File handling. It also includes an extra guide, still in development, about [Meta-Programming in Elixir](https://hexdocs.pm/elixir/quote-and-unquote.html); -* Elixir v0.13 provides a new comprehension syntax that not only works with lists, but with any [`Enumerable`](/docs/stable/elixir/Enumerable.html). The output of a comprehension is also extensible via the [`Collectable`](/docs/stable/elixir/Collectable.html) protocol; +* Elixir v0.13 provides a new comprehension syntax that not only works with lists, but with any [`Enumerable`](https://hexdocs.pm/elixir/Enumerable.html). The output of a comprehension is also extensible via the [`Collectable`](https://hexdocs.pm/elixir/Collectable.html) protocol; * Mix, Elixir's build tool, has been improved in order to provide better workflows when compiling projects and working with dependencies; -* There are many other changes, like the addition of [StringIO](/docs/stable/elixir/StringIO.html), support for [tags and filters in ExUnit](/docs/stable/ex_unit/ExUnit.Case.html) and more. Please check the [CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.13.0/CHANGELOG.md) for the complete list. +* There are many other changes, like the addition of [StringIO](https://hexdocs.pm/elixir/StringIO.html), support for [tags and filters in ExUnit](https://hexdocs.pm/ex_unit/ExUnit.Case.html) and more. Please check the [CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.13.0/CHANGELOG.md) for the complete list. -Even with all those improvements, Elixir v0.13.0 is backwards compatible with Elixir v0.12.5 and upgrading should be a clean process. +Even with all those improvements, Elixir v0.13.0 is backwards compatible with Elixir v0.12.5 and upgrading should be a clean process. ## Maps Maps are key-value data structures: -```iex +```elixir iex> map = %{"hello" => :world} %{"hello" => :world} iex> map["hello"] @@ -55,7 +56,7 @@ Maps do not have a explicit ordering and keys and values can be any term. Maps can be pattern matched on: -```iex +```elixir iex> %{"hello" => world} = map %{"hello" => :world} iex> world @@ -68,11 +69,11 @@ iex> %{"other" => value} = map A map pattern will match any map that has all the keys specified in the pattern. The values for the matching keys must also match. For example, `%{"hello" => world}` will match any map that has the key `"hello"` and assign the value to `world`, while `%{"hello" => "world"}` will match any map that has the key `"hello"` with value equals to `"world"`. An empty map pattern (`%{}`) will match all maps. -Developers can use the functions in the [`Map` module](/docs/stable/elixir/Map.html) to work with maps. For more information on maps and how they compare to other associative data structures in the language, please check the [Maps chapter in our new Getting Started guide](/getting_started/7.html). Elixir Sips has also released two episodes that cover maps ([part 1](http://elixirsips.com/episodes/054_maps_part_1.html) and [part 2](http://elixirsips.com/episodes/055_maps_part_2.html)). +Developers can use the functions in the [`Map` module](https://hexdocs.pm/elixir/Map.html) to work with maps. For more information on maps and how they compare to other associative data structures in the language, please check the [Maps chapter in our new Getting Started guide](https://hexdocs.pm/elixir/keywords-and-maps.html). Elixir Sips has also released two episodes that cover maps ([part 1](http://elixirsips.com/episodes/054_maps_part_1.html) and [part 2](http://elixirsips.com/episodes/055_maps_part_2.html)). Maps also provide special syntax for creating, accessing and updating maps with atom keys: -```iex +```elixir iex> user = %{name: "john", age: 27} %{name: "john", age: 27} iex> user.name @@ -85,7 +86,7 @@ iex> user.name Both access and update syntax above expect the given keys to exist. Trying to access or update a key that does not exist raises an error: -```iex +```elixir iex> %{ user | address: [] } ** (ArgumentError) argument error :maps.update(:address, [], %{}) @@ -110,7 +111,7 @@ Internally, this record is represented as the following tuple: Records can also be created and pattern matched on: -```iex +```elixir iex> user = User[name: "john"] User[name: "john", age: 0] iex> user.name @@ -156,21 +157,21 @@ end Now a `User` struct can be created without a need to explicitly list all necessary fields: -```iex +```elixir iex> user = %User{name: "john"} %User{name: "john", age: 0} ``` Trying to create a struct with an unknown key raises an error during compilation: -```iex +```elixir iex> user = %User{address: []} ** (CompileError) unknown key :address for struct User ``` Furthermore, every struct has a `__struct__` field which contains the struct name: -```iex +```elixir iex> user.__struct__ User ``` @@ -179,15 +180,15 @@ The `__struct__` field is also used for polymorphic dispatch in protocols, addre It is interesting to note that structs solve both drawbacks we have earlier mentioned regarding records. Structs are purely data and polymorphic dispatch is now faster and more robust as it happens only for explicitly tagged structs. -For more information on structs, check out the [Structs chapter in the getting started guide](/getting_started/15.html) (you may also want to read the new Protocols chapter after it). +For more information on structs, check out the [Structs chapter in the getting started guide](https://hexdocs.pm/elixir/structs.html) (you may also want to read the new [Protocols chapter](https://hexdocs.pm/elixir/protocols.html) after it). ## Maps, structs and the future With the introduction of maps and structs, some deprecations will arrive on upcoming releases. First of all, the `ListDict` data structure is being deprecated and phased out. Records are also being deprecated from the language, although it is going to be a longer process, as many projects and Elixir itself still use records in diverse occasions. -Note though only Elixir records are being deprecated. Erlang records, which are basically syntax sugar around tuples, will remain in the language for the rare cases Elixir developers need to interact with Erlang libraries that provide records. In particular, the [Record](/docs/stable/elixir/Record.html) has been updated to provide the new Record API (while keeping the old one for backwards compatibility). +Note though only Elixir records are being deprecated. Erlang records, which are basically syntax sugar around tuples, will remain in the language for the rare cases Elixir developers need to interact with Erlang libraries that provide records. In particular, the [Record](https://hexdocs.pm/elixir/Record.html) has been updated to provide the new Record API (while keeping the old one for backwards compatibility). -Finally, structs are still in active development and new features, like `@derive`, should land in upcoming Elixir releases. For those interested, the [original maps and structs proposal is still availble](https://gist.github.com/josevalim/b30c881df36801611d13). +Finally, structs are still in active development and new features, like `@derive`, should land in upcoming Elixir releases. For those interested, the [original maps and structs proposal is still available](https://gist.github.com/josevalim/b30c881df36801611d13). ## Comprehensions @@ -195,38 +196,38 @@ Erlang R17 also introduced recursion to anonymous functions. This feature, while The most common use case of a comprehension are [list comprehensions](https://en.wikipedia.org/wiki/List_comprehension). For example, we can get all the square values of elements in a list as follows: -```iex +```elixir iex> for n <- [1, 2, 3, 4], do: n * n [1, 4, 9, 16] ``` We say the `n <- [1, 2, 3, 4]` part is a comprehension generator. In previous Elixir versions, Elixir supported only lists in generators. In Elixir v0.13.0, any Enumerable is supported (ranges, maps, etc): -```iex +```elixir iex> for n <- 1..4, do: n * n [1, 4, 9, 16] ``` As in previous Elixir versions, there is also support for a bitstring generator. In the example below, we receive a stream of RGB pixels as a binary and break it down into triplets: -```iex +```elixir iex> pixels = <<213, 45, 132, 64, 76, 32, 76, 0, 0, 234, 32, 15>> iex> for <>, do: {r, g, b} -[{213,45,132}, {64,76,32}, {76,0,0}, {234,32,15}] +[{213, 45, 132}, {64, 76, 32}, {76, 0, 0}, {234, 32, 15}] ``` By default, a comprehension returns a list as a result. However the result of a comprehension can be inserted into different data structures by passing the `:into` option. For example, we can use bitstring generators with the `:into` option to easily remove all spaces in a string: -```iex +```elixir iex> for <>, c != ?\s, into: "", do: <> "helloworld" ``` -Sets, maps and other dictionaries can also be given with the `:into` option. In general, the `:into` accepts any structure as long as it implements the [`Collectable` protocol](/docs/stable/elixir/Collectable.html). +Sets, maps and other dictionaries can also be given with the `:into` option. In general, the `:into` accepts any structure as long as it implements the [`Collectable` protocol](https://hexdocs.pm/elixir/Collectable.html). For example, the `IO` module provides streams, that are both `Enumerable` and `Collectable`. You can implement an echo terminal that returns whatever is typed into the shell, but in upcase, using comprehensions: -```iex +```elixir iex> stream = IO.stream(:stdio, :line) iex> for line <- stream, into: stream do ...> String.upcase(line) <> "\n" @@ -237,18 +238,18 @@ This makes comprehensions useful not only for working with in-memory collections ## Mix workflows -The last big change we want to discuss in this release are the improvements done to Mix, Elixir's build tool. Mix is an essential tool to Elixir developers and helps developers to compile their projects, manage their dependencies, run tests and so on. +The last big change we want to discuss in this release are the improvements done to Mix, Elixir's build tool. Mix is an essential tool to Elixir developers and helps developers to compile their projects, manage their dependencies, run tests and so on. In previous releases, Mix was used to download and compile dependencies per environment. That meant the usual workflow was less than ideal: every time a dependency was updated, developers had to explicitly fetch and compile the dependencies for each environment. The workflow would be something like: -``` +```bash $ mix deps.get $ mix compile $ MIX_ENV=test mix deps.get $ mix test ``` -In Elixir v0.13, `mix deps.get` only fetches dependencies and it does so accross all environments (unless an `--only` flag is specified). To support this new behaviour, dependencies now support the `:only` option: +In Elixir v0.13, `mix deps.get` only fetches dependencies and it does so across all environments (unless an `--only` flag is specified). To support this new behaviour, dependencies now support the `:only` option: ```elixir def deps do @@ -261,7 +262,7 @@ Dependencies now are also automatically compiled before you run a command. For e ## hex.pm -This release also marks the announcement of [hex.pm](http://hex.pm/), a package manager for the Erlang VM. Hex allows you to package and publish your projects while fetching them and performing dependency resolution in your applications. +This release also marks the announcement of [hex.pm](https://hex.pm/), a package manager for the Erlang VM. Hex allows you to package and publish your projects while fetching them and performing dependency resolution in your applications. Currently Hex only integrates with Mix and contributions to extend it to other tools and other languages in the Erlang VM are welcome! @@ -269,14 +270,14 @@ Currently Hex only integrates with Mix and contributions to extend it to other t As seen in this announcement, this release dictates many of the developments that will happen in Elixir and its community in the following weeks. All projects are recommended to start moving from records to structs, paving the way for the deprecation of records before 1.0. -The next months will also focus on integrating Elixir more tightly to OTP. During the keynote at Erlang Factory, [Catalyse Change](http://www.youtube.com/watch?v=Djv4C9H9yz4), Dave Thomas and I argued that there are many useful patterns, re-implemented everyday by developers, that could make development more productive within the Erlang VM if exposed accordingly. +The next months will also focus on integrating Elixir more tightly to OTP. During the keynote at Erlang Factory, [Catalyse Change](https://www.youtube.com/watch?v=Djv4C9H9yz4), Dave Thomas and I argued that there are many useful patterns, re-implemented everyday by developers, that could make development more productive within the Erlang VM if exposed accordingly. That said, in the next months we plan to: * Integrate applications configuration (provided by OTP) right into Mix; * Provide an Elixir logger that knows how to print and format Elixir exceptions and stacktraces; -* Properly expose the functionality provided by Applications, Supervisors, GenServers and GenEvents and study how they can integrate with Elixir. For example, how to consume events from GenEvent as a [stream of data](/docs/stable/elixir/Stream.html)? -* Study how patterns like tasks and agents can be integrated into the language, often picking up the lessons learned by libraries like [e2](http://e2project.org/erlang.html) and [functionality exposed by OTP itself](http://erlang.org/doc/man/rpc.html); +* Properly expose the functionality provided by Applications, Supervisors, GenServers and GenEvents and study how they can integrate with Elixir. For example, how to consume events from GenEvent as a [stream of data](https://hexdocs.pm/elixir/Stream.html)? +* Study how patterns like tasks and agents can be integrated into the language, often picking up the lessons learned by libraries like [e2](http://e2project.org/erlang.html) and [functionality exposed by OTP itself](http://www.erlang.org/doc/man/rpc.html); * Rewrite the Mix and ExUnit guides to focus on applications and OTP as a whole, rebranding it to "Building Apps with Mix and OTP"; -You can learn more about Elixir in our [Getting Started guide](/getting_started/1.html) and download this release in the [v0.13 announcement](https://github.com/elixir-lang/elixir/releases/tag/v0.13.0). We hope to see you at [ElixirConf](http://elixirconf.com/) as well as pushing your packages to [hex.pm](http://hex.pm/). +You can learn more about Elixir in our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) and download this release in the [v0.13 announcement](https://github.com/elixir-lang/elixir/releases/tag/v0.13.0). We hope to see you at [ElixirConf](http://elixirconf.com/) as well as pushing your packages to [hex.pm](https://hex.pm/). diff --git a/_posts/2014-06-17-elixir-v0-14-0-released.markdown b/_posts/2014-06-17-elixir-v0-14-0-released.markdown index 707f047a3..cfa23e5ca 100644 --- a/_posts/2014-06-17-elixir-v0-14-0-released.markdown +++ b/_posts/2014-06-17-elixir-v0-14-0-released.markdown @@ -1,7 +1,8 @@ --- layout: post title: Elixir v0.14.0 released -author: José Valim +authors: +- José Valim category: Releases excerpt: "Elixir v0.14.0 is released and expands the work on structs and bringing more of OTP right into Elixir and Mix" --- @@ -40,7 +41,7 @@ Structs allow us to provide default values for a map fields. Structs also valida #=> ** (CompileError) iex:2: unknown key :unknown for struct User ``` -We say structs are *raw* because they do not implement any of the protocols that are implemented for maps by default. For instance, we can call `Enum.each/2` for a map, which uses the [`Enumerable`](/docs/stable/elixir/Enumerable.html) protocol: +We say structs are *raw* because they do not implement any of the protocols that are implemented for maps by default. For instance, we can call `Enum.each/2` for a map, which uses the [`Enumerable`](https://hexdocs.pm/elixir/Enumerable.html) protocol: ```elixir Enum.each %{foo: :bar}, fn {k, v} -> @@ -134,32 +135,32 @@ update_in dungeon.rooms[room_id].users[user_id].device_codes, &Set.delete(&1, co put_in dungeon, [:rooms, room_id, :users, user_id, :username], "new username" ``` -You can read more information about nested access in [the Access protocol documentation](/docs/stable/elixir/Access.html) and in the docs for [`get_in/2`](/docs/stable/elixir/Kernel.html#get_in/2) and friends. +You can read more information about nested access in [the Access protocol documentation](https://hexdocs.pm/elixir/Access.html) and in the docs for [`get_in/2`](https://hexdocs.pm/elixir/Kernel.html#get_in/2) and friends. ## Mix and OTP OTP is a set of libraries that ships with Erlang. Erlang developers use OTP to build robust, fault-tolerant applications. -In v0.14.0, Elixir closely integrates with OTP by providing modules for building [servers](/docs/stable/elixir/GenServer.html), [event managers and event handlers](/docs/stable/elixir/GenEvent.html), [supervisors](/docs/stable/elixir/Supervisor.html) and [applications](/docs/stable/elixir/Application.html). +In v0.14.0, Elixir closely integrates with OTP by providing modules for building [servers](https://hexdocs.pm/elixir/GenServer.html), [supervisors](https://hexdocs.pm/elixir/Supervisor.html) and [applications](https://hexdocs.pm/elixir/Application.html). -We have also introduced the concepts of [agents](/docs/stable/elixir/Agent.html) and the idea of [tasks](/docs/stable/elixir/Task.html), which can be supervised and distributed. Application configuration has been made first class in Mix, allowing developers to configure their dependencies, sometimes even using different configurations per environment (dev, test or prod by default). +We have also introduced the concepts of [agents](https://hexdocs.pm/elixir/Agent.html) and the idea of [tasks](https://hexdocs.pm/elixir/Task.html), which can be supervised and distributed. Application configuration has been made first class in Mix, allowing developers to configure their dependencies, sometimes even using different configurations per environment (dev, test or prod by default). -This functionality is at the core of building applications in Erlang and Elixir. For this reason we have published a new guide called [Mix and OTP](/getting_started/mix_otp/1.html) where we build a distributed key-value store to help explore all concepts mentioned above. The guide is quite fresh, so please do submit pull requests for typos and mistakes. Feedback is also welcome! +This functionality is at the core of building applications in Erlang and Elixir. For this reason we have published a new guide called [Mix and OTP](https://hexdocs.pm/elixir/introduction-to-mix.html) where we build a distributed key-value store to help explore all concepts mentioned above. The guide is quite fresh, so please do submit pull requests for typos and mistakes. Feedback is also welcome! -Note "Mix and OTP" is our most advanced guide so far and it expects you to have read our introductory guide. In case you haven't yet, you can [get started here](/getting_started/1.html). +Note "Mix and OTP" is our most advanced guide so far and it expects you to have read our introductory guide. In case you haven't yet, you can [get started here](https://hexdocs.pm/elixir/introduction.html). ## What's next? -With v0.14.0 we have reached many of the milestones [we have set in the previous release](/blog/2014/04/21/elixir-v0-13-0-released/#toc_8). This brings us closer to Elixir v1.0.0 and only a handful of tasks are pending: +With v0.14.0 we have reached many of the milestones [we have set in the previous release](/blog/2014/04/21/elixir-v0-13-0-released/#the-next-steps). This brings us closer to Elixir v1.0 and only a handful of tasks are pending: * Provide an Elixir logger that knows how to print and format Elixir exceptions and stacktraces. Work has already started on this front as Elixir already prints errors coming from the application startup nicely; -* Continue the work of cleaning up the [Kernel module](/docs/stable/elixir/Kernel.html). In v0.14.0, we added alternatives for conversion functions, like `integer_to_binary/1` to `Integer.to_string/1`, now they must be properly deprecated and removed; +* Continue the work of cleaning up the [Kernel module](https://hexdocs.pm/elixir/Kernel.html). In v0.14.0, we added alternatives for conversion functions, like `integer_to_binary/1` to `Integer.to_string/1`, now they must be properly deprecated and removed; * Support mix aliases, allowing developers to easily define Mix shortcuts for their favorite tasks; -* Solve all remaining [open issues](https://github.com/elixir-lang/elixir/issues?state=open). We have always kept the issues tracker tidy and there is little work left to solve the existing issues. Note we have also listed all [upcoming backwards incompatible changes](https://github.com/elixir-lang/elixir/issues?labels=Note%3ABackwards+incompatible&page=1&state=open). Many of those changes will actually be deprecated first and developers should be able to follow along without breaking changes in minor releases, but they are breaking changes in the sense they work in v0.14.0 but will work differently by the time v1.0.0 is released; +* Solve all remaining [open issues](https://github.com/elixir-lang/elixir/issues?state=open). We have always kept the issues tracker tidy and there is little work left to solve the existing issues. Note we have also listed all [upcoming backwards incompatible changes](https://github.com/elixir-lang/elixir/issues?labels=Note%3ABackwards+incompatible&page=1&state=open). Many of those changes will actually be deprecated first and developers should be able to follow along without breaking changes in minor releases, but they are breaking changes in the sense they work in v0.14.0 but will work differently by the time v1.0 is released; -That's all for now! Elixir developers can see [a summary of all changes in v0.14.0 in the release notes](https://github.com/elixir-lang/elixir/releases/tag/v0.14.0). In case you are new around here, you can get started with Elixir by reading [our Getting Started guide](/getting_started/1.html). +That's all for now! Elixir developers can see [a summary of all changes in v0.14.0 in the release notes](https://github.com/elixir-lang/elixir/releases/tag/v0.14.0). In case you are new around here, you can get started with Elixir by reading [our Getting Started guide](https://hexdocs.pm/elixir/introduction.html). We hope to see you all this July at [ElixirConf](http://elixirconf.com/)! diff --git a/_posts/2014-08-07-elixir-v0-15-0-released.markdown b/_posts/2014-08-07-elixir-v0-15-0-released.markdown index 5a7e1ccc3..32b579555 100644 --- a/_posts/2014-08-07-elixir-v0-15-0-released.markdown +++ b/_posts/2014-08-07-elixir-v0-15-0-released.markdown @@ -1,22 +1,23 @@ --- layout: post title: Elixir v0.15.0 released -author: José Valim +authors: +- José Valim category: Releases -excerpt: "Elixir v0.15.0 introduces Elixir's Logger, Mix aliases and is the last stop before Elixir v1.0.0. We are also glad to welcome Alexei into our team!" +excerpt: "Elixir v0.15.0 introduces Elixir's Logger, Mix aliases and is the last stop before Elixir v1.0. We are also glad to welcome Alexei into our team!" --- Hello everyone! -We are glad to announce v0.15.0 has been released. We have spent the last 2 months tidying up the existing APIs, ensuring consistency, improving performance and more. As a result, v0.15.0 is the last minor branch before Elixir v1.0.0! +We are glad to announce v0.15.0 has been released. We have spent the last 2 months tidying up the existing APIs, ensuring consistency, improving performance and more. As a result, v0.15.0 is the last minor branch before Elixir v1.0! -There are also no more planned deprecations nor backward incompatible changes which means it is extremely likely that code that runs on v0.15.0 will run exactly the same on Elixir v1.0.0. +There are also no more planned deprecations nor backward incompatible changes which means it is extremely likely that code that runs on v0.15.0 will run exactly the same on Elixir v1.0. If you are interested in the specific details for this release, [please check our CHANGELOG](https://github.com/elixir-lang/elixir/blob/v0.15.0/CHANGELOG.md). In this post, we will focus on three new features in this release: Logger, Mix aliases, the fresh Elixir Web Installer for Windows, and share some exciting news at the end! ## Logger -Elixir now ships with a new application called logger. This application provides [the Logger module](http://elixir-lang.org/docs/master/logger/Logger.html), which is the main API developers will use for logging: +Elixir now ships with a new application called logger. This application provides [the Logger module](https://hexdocs.pm/logger/Logger.html), which is the main API developers will use for logging: ```elixir require Logger @@ -29,7 +30,7 @@ By default, the code above will log the following message to your console: 10:27:39.083 [debug] hello ``` -Logger provides multiple backends to where messages are logged. For now Elixir ships only with a console backend but there are developers already working on file (with support to external log rotation) and [syslog](http://en.wikipedia.org/wiki/Syslog) backends. +Logger provides multiple backends to where messages are logged. For now Elixir ships only with a console backend but there are developers already working on file (with support to external log rotation) and [syslog](https://en.wikipedia.org/wiki/Syslog) backends. When we started Logger, the main objective was to translate Erlang messages into Elixir, so terms are formatted in Elixir syntax. Before this release, the following code @@ -70,11 +71,11 @@ Function: #Function<20.90072148/0 in :erl_eval.expr/5> As soon as we started working on Logger, we realized we could go further than simply translating Erlang messages and provide a fully featured logger library. At this moment, Logger also supports: * 4 log levels: debug, info, warn and error - * Custom formatting: you can specify a format string that tells exactly how messages should be logged. The default string is: "$time $metadata[$level] $message\n" but [many attributes are supported](http://elixir-lang.org/docs/master/logger/Logger.Formatter.html) + * Custom formatting: you can specify a format string that tells exactly how messages should be logged. The default string is: "$time $metadata[$level] $message\n" but [many attributes are supported](https://hexdocs.pm/logger/Logger.Formatter.html) * Custom translators: so you can translate log messages coming from any Erlang application into Elixir syntax * Metadata: metadata allows developers to store information in the current process that will be available to all logged messages. For example, a web application can generate a `request_id`, store it as metadata, and all messages logged during that request will be properly identified with `request_id=...` in the log -We have also relied a lot on the [research and work done by Andrew Thompson and the folks at Basho behind Lager](http://www.youtube.com/watch?v=8BNpOHFvg_Q) to ensure our logger is performant and robust. On this front, Logger +We have also relied a lot on the [research and work done by Andrew Thompson and the folks at Basho behind Lager](https://www.youtube.com/watch?v=8BNpOHFvg_Q) to ensure our logger is performant and robust. On this front, Logger * alternates between sync and async modes when logging messages to keep it performant when required but also apply back-pressure when under stress * formats and truncates messages on the client to avoid clogging the backends @@ -136,13 +137,13 @@ In other words, aliases can be three different structures: 2. An anonymous function (that is invoked passing the task arguments) 3. A list containing strings or anonymous functions -You can find more information about aliases by reading the [Mix documentation](http://elixir-lang.org/docs/master/mix/) (there is a section about Aliases around the middle). +You can find more information about aliases by reading the [Mix documentation](https://hexdocs.pm/mix/) (there is a section about Aliases around the middle). We also would like to thank [Anthony Grimes](https://github.com/raynes) for the support and [Phil Halgelberg](https://github.com/technomancy) for [the work on Lein](https://github.com/technomancy/leiningen) which Mix borrows a lot from. ## Elixir Web Installer for Windows -At the beginning of this summer, [Chris Hyndman](http://github.com/chyndman) joined us as a Google Summer of Code student to help us improve the Elixir story on Windows. Chris has been essential in: +At the beginning of this summer, [Chris Hyndman](https://github.com/chyndman) joined us as a Google Summer of Code student to help us improve the Elixir story on Windows. Chris has been essential in: * Guaranteeing our test suite is green on Windows, fixing many bugs in the process; * [Documenting how to compile Elixir from source on Windows](https://github.com/elixir-lang/elixir/wiki/Windows) @@ -150,7 +151,7 @@ At the beginning of this summer, [Chris Hyndman](http://github.com/chyndman) joi Chris has also built an [Elixir Web Installer for Windows](https://github.com/elixir-lang/elixir-windows-setup). The web installer checks all available Elixir versions and allows you to pick which one to install. It will also fetch and install Erlang in your machine in case it has not been installed yet. -If you want to give Elixir and the Web Installer a try, you can [download the current version here](http://s3.hex.pm/elixir-websetup.exe). And, if [Chocolatey](https://chocolatey.org/) is your thing, remember you can also install Elixir on Windows by running `cinst elixir`. +If you want to give Elixir and the Web Installer a try, you can [download the current version here](https://repo.hex.pm/elixir-websetup.exe). And, if [Chocolatey](https://chocolatey.org/) is your thing, remember you can also install Elixir on Windows by running `cinst elixir`. ## Welcome Alexei! @@ -160,6 +161,6 @@ Alexei is also interested in how we can extend our tooling to the Erlang ecosyst ## What's next? -We are very close to launch Elixir v1.0.0! All planned features are already in Elixir's codebase and at the moment there are only [four open issues in our tracker tagged with the v1.0 milestone](https://github.com/elixir-lang/elixir/issues?q=is%3Aopen+is%3Aissue+milestone%3Av1.0). +We are very close to launch Elixir v1.0! All planned features are already in Elixir's codebase and at the moment there are only [four open issues in our tracker tagged with the v1.0 milestone](https://github.com/elixir-lang/elixir/issues?q=is%3Aopen+is%3Aissue+milestone%3Av1.0). -Our estimated date for the first release candidate for Elixir v1.0.0 is August 30th. This means there is no better time to learn Elixir than now! If you haven't started yet, you can get started with Elixir by reading [our Getting Started guide](/getting_started/1.html) or by checking one of the many "Learning Resources" on the sidebar. +Our estimated date for the first release candidate for Elixir v1.0 is August 30th. This means there is no better time to learn Elixir than now! If you haven't started yet, you can get started with Elixir by reading [our Getting Started guide](https://hexdocs.pm/elixir/introduction.html) or by checking one of the many "Learning Resources" on the sidebar. diff --git a/_posts/2014-09-18-elixir-v1-0-0-released.markdown b/_posts/2014-09-18-elixir-v1-0-0-released.markdown index 8ce7c9493..72929864c 100644 --- a/_posts/2014-09-18-elixir-v1-0-0-released.markdown +++ b/_posts/2014-09-18-elixir-v1-0-0-released.markdown @@ -1,14 +1,13 @@ --- layout: post -title: Elixir v1.0.0 released -author: José Valim +title: Elixir v1.0 released +authors: +- José Valim category: Releases -excerpt: Elixir v1.0.0 is finally out! +excerpt: Elixir v1.0 is finally out! --- -Hello everyone! - -We are glad to announce Elixir v1.0.0 is finally out. It has been 8005 commits [by 189 contributors](https://github.com/elixir-lang/elixir/graphs/contributors), including the initial commit on [January 9th, 2011](https://github.com/elixir-lang/elixir/commit/337c3f2d569a42ebd5fcab6fef18c5e012f9be5b)! +We are glad to announce Elixir v1.0 is finally out. It has been 8005 commits [by 189 contributors](https://github.com/elixir-lang/elixir/graphs/contributors?from=2011-01-12&to=2014-09-10&type=c), including the initial commit on [January 9th, 2011](https://github.com/elixir-lang/elixir/commit/337c3f2d569a42ebd5fcab6fef18c5e012f9be5b)! ## What's Elixir? @@ -18,45 +17,45 @@ Elixir leverages the Erlang VM, known for running low-latency, distributed and f ## What's new? -This release is the consolidation of all the work done through the years. With v1.0.0, we have reached a stable milestone for the growth of software and projects written in Elixir. +This release is the consolidation of all the work done through the years. With v1.0, we have reached a stable milestone for the growth of software and projects written in Elixir. -Elixir will follow [semantic versioning](http://semver.org), which means code written for Elixir v1.0.0 will continue to compile and run correctly for all versions under the v1 branch (e.g. v1.0.1, v1.1.0, v1.2.0). +Elixir will follow [semantic versioning](http://semver.org), which means code written for Elixir v1.0 will continue to compile and run correctly for all versions under the v1 branch (e.g. v1.0.1, v1.1.0, v1.2.0). Elixir is composed of 6 applications, all under the same versioning constraints: - * [Elixir](/docs/stable/elixir) - the Elixir compiler, runtime and the standard library - * [EEx](/docs/stable/eex) - Elixir's templating library, useful for generating any kind of document dynamically - * [ExUnit](/docs/stable/ex_unit) - Elixir's unit test library, with support for concurrent testing, custom formatters, filters and much more - * [IEx](/docs/stable/iex) - Elixir's interactive shell with code reloading, auto-complete, and easy access to documentation, typespecs and more - * [Logger](/docs/stable/logger) - the latest addition to the group, Logger provides reliable logging and configurable backends (with syslog, file and many other backends provided by the community) - * [Mix](/docs/stable/mix) - Elixir's build tool that generates, compiles and tests projects as well as manages your dependencies + * [Elixir](https://hexdocs.pm/elixir/) - the Elixir compiler, runtime and the standard library + * [EEx](https://hexdocs.pm/eex/) - Elixir's templating library, useful for generating any kind of document dynamically + * [ExUnit](https://hexdocs.pm/ex_unit/) - Elixir's unit test library, with support for concurrent testing, custom formatters, filters and much more + * [IEx](https://hexdocs.pm/iex/) - Elixir's interactive shell with code reloading, auto-complete, and easy access to documentation, typespecs and more + * [Logger](https://hexdocs.pm/logger/) - the latest addition to the group, Logger provides reliable logging and configurable backends (with syslog, file and many other backends provided by the community) + * [Mix](https://hexdocs.pm/mix/) - Elixir's build tool that generates, compiles and tests projects as well as manages your dependencies -With v1.0.0, we are providing a stable platform for the community to leverage and extend, and we are extremely excited with the projects and possibilities that are ahead of us! +With v1.0, we are providing a stable platform for the community to leverage and extend, and we are extremely excited with the projects and possibilities that are ahead of us! -We hope the [Hex package manager](http://hex.pm) will be the home of many of those projects and remember the whole Erlang ecosystem is also available to Elixir developers. +We hope the [Hex package manager](https://hex.pm) will be the home of many of those projects and remember the whole Erlang ecosystem is also available to Elixir developers. ## Expectations -We would like to elaborate on the expectations regarding Elixir v1.0.0. Although we expect that the vast majority of programs will remain compatible over time, it is impossible to guarantee that no future change will break any program. +We would like to elaborate on the expectations regarding Elixir v1.0. Although we expect that the vast majority of programs will remain compatible over time, it is impossible to guarantee that no future change will break any program. Under some unlikely circumstances, we may introduce changes that break existing code: * Security: a security issue in the implementation may arise whose resolution requires backwards incompatible changes. We reserve the right to address such security issues. - * Bugs: if an application has undesired behaviour, a program that depends on the buggy behavior may break if the bug is fixed. We reserve the right to fix such bugs. + * Bugs: if an application has undesired behaviour, a program that depends on the buggy behaviour may break if the bug is fixed. We reserve the right to fix such bugs. - * Compiler front-end: improvements may be done to the compiler, introducing new warnings for ambiguous modes and providing more detailed error messages. Those can lead to compilation errors (when running with `--warning-as-errors`) or tooling failures when expecting specific messages (although one should avoid such). We reserve the right to do such improvements. + * Compiler front-end: improvements may be done to the compiler, introducing new warnings for ambiguous modes and providing more detailed error messages. Those can lead to compilation errors (when running with `--warning-as-errors`) or tooling failures when asserting on specific error messages (although one should avoid such). We reserve the right to do such improvements. * Imports: new functions may be added to the Kernel module, which is auto-imported. They may collide with local functions defined in your modules. Collisions can be resolved in a backwards compatible fashion using `import Kernel, except: [...]` with a list of all functions you don't want imported from Kernel. We reserve the right to do such additions. -Elixir binaries are guaranteed to be backwards compatible within the same branch. Code compiled with v1.0.0 shall work with Elixir v1.0.1 runtime but code compiled with Elixir v1.0.0 should be recompiled to work with Elixir v1.1.0 onwards. +Elixir binaries are guaranteed to be backwards compatible within the same branch. Code compiled with v1.0 shall work with Elixir v1.0.1 runtime but code compiled with Elixir v1.0 should be recompiled to work with Elixir v1.1.0 onwards. These expectations also apply to future releases under the v1 branch, except for experimental features, which will be explicitly marked as such and not provide any compatibility guarantee until they are stabilized. ## Learn more -You can get started with Elixir via our [Getting Started guide](/getting_started/1.html). There are quite some Elixir books out there too, now getting sent to the presses, quite a few can be found in the sidebar, which also includes screencasts and other resources. +You can get started with Elixir via our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html). There are quite some Elixir books out there too, now getting sent to the presses, quite a few can be found in the sidebar, which also includes screencasts and other resources. You can also learn more about Elixir by checking out [the videos from ElixirConf 2014](http://www.confreaks.com/events/elixirconf2014), the first (and so far the best) Elixir conference ever! You can learn more about [the language history](http://www.confreaks.com/videos/4134-elixirconf2014-keynote-elixir), [how Elixir can change the way you code](http://www.confreaks.com/videos/4119-elixirconf2014-opening-keynote-think-different) or [even hear stories of how Elixir is being used in production](http://www.confreaks.com/videos/4131-elixirconf2014-otp-in-production-the-nitty-gritty-details-of-game-servers). -Finally, by popular demand, we have [released some Elixir stickers](http://www.stickermule.com/user/1070631438/stickers), which are available with a discounted price to celebrate v1.0.0! +Finally, by popular demand, we have [released some Elixir stickers](http://www.stickermule.com/user/1070631438/stickers), which are available with a discounted price to celebrate v1.0! diff --git a/_posts/2015-09-28-elixir-v1-1-0-released.markdown b/_posts/2015-09-28-elixir-v1-1-0-released.markdown new file mode 100644 index 000000000..5adecea87 --- /dev/null +++ b/_posts/2015-09-28-elixir-v1-1-0-released.markdown @@ -0,0 +1,28 @@ +--- +layout: post +title: Elixir v1.1 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.1 brings enhancements, bug fixes, performance improvements and more into Elixir. +--- + +Elixir v1.1 has been released and it brings enhancements, bug fixes, performance improvements and more into Elixir. + +Elixir v1.1 supports both Erlang 17 and Erlang 18. This is, however, the last release supporting Erlang 17, so upgrading to Erlang 18 is advised. Elixir v1.2 will introduce features that are Erlang 18 only. + +On the standard library side, about 40 new functions have been added to Elixir public APIs. For example, [`Enum`](/docs/v1.1/elixir/Enum.html) got [`dedup/1`](/docs/v1.1/elixir/Enum.html#dedup/1), [`random/1`](/docs/v1.1/elixir/Enum.html#random/1), and a couple more. The [`String`](/docs/v1.1/elixir/String.html) module can now [calculate the distance between strings](/docs/v1.1/elixir/String.html#jaro_distance/2). We use this feature to provide suggestions whenever an unknown task name is given when using Mix. You can also [yield to](/docs/v1.1/elixir/Task.html#yield/2) and [shutdown](/docs/v1.1/elixir/Task.html#shutdown/2) tasks in the [`Task`](/docs/v1.1/elixir/Task.html) module. + +The applications that ship with Elixir also has seen improvements and bug fixes. [ExUnit](/docs/v1.1/ex_unit/ExUnit.html), Elixir's test framework, now has support for skipping tests via tags `@tag :skip`, as well as the ability to capture logs via `@tag :capture_log`, ensuring that all log messages during the tests are captured. Even better, in case of failures, all captured log messages are printed along-side the test error report. + +Mix ships with a [`mix profile.fprof`](/docs/v1.1/mix/Mix.Tasks.Profile.Fprof.html), useful for profiling your application code. The [`mix app.start`](/docs/v1.1/mix/Mix.Tasks.App.Start.html) has also been publicly documented. Although you likely won't invoke it directly through the command line, it is useful when writing your own tasks that require the current application to be up and running. + +Mix also provides faster re-compilation times. Every time you compile your Elixir code, Mix generates a graph of the dependencies between source files. For example, if `a.ex` depends on `b.ex`, every time `b.ex` changes, `a.ex` must be recompiled. Elixir v1.1 improves this tracking by separating compile-time dependencies from runtime ones, recompiling a file only if a compile-time dependency changed. In projects that have a main dispatch entity, like a web-app router, we have seen `mix compile` go from recompiling the whole project to one or two files per run. + +We have also seen great progress on areas that go beyond the source code. In particular, we have added a [CODE\_OF\_CONDUCT.md](https://github.com/elixir-lang/elixir/blob/v1.1/CODE_OF_CONDUCT.md) to guarantee our community continues to grow into a safe and welcoming place for everyone. + +We have also released a new ExDoc version. It provides a [beautiful, clean and readable way to navigate the Elixir documentation](https://hexdocs.pm/elixir/1.1.0) and it is available to any Elixir project. The latest version includes initial support for User Guides and we have more features and improvements coming on the way. + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.1.0). Don't forget to check [the Install section](/install.html) to get Elixir installed. + +Happy coding! diff --git a/_posts/2016-01-03-elixir-v1-2-0-released.markdown b/_posts/2016-01-03-elixir-v1-2-0-released.markdown new file mode 100644 index 000000000..1b8a7efd8 --- /dev/null +++ b/_posts/2016-01-03-elixir-v1-2-0-released.markdown @@ -0,0 +1,118 @@ +--- +layout: post +title: Elixir v1.2 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.2 brings enhancements, bug fixes, performance improvements and more into Elixir. +--- + +v1.2 brings enhancements, bug fixes, performance improvements and more into Elixir. Elixir v1.2 relies on many features in Erlang 18, requiring at least Erlang 18+. Upgrading to Erlang 18 is therefore necessary before upgrading Elixir. + +To celebrate this release and the new year, we have also reviewed both the [Getting Started](https://hexdocs.pm/elixir/introduction.html) and [Mix & OTP](https://hexdocs.pm/elixir/introduction-to-mix.html) guides, bringing it up to date and exploring new functionalities added since Elixir v1.0. + +## Erlang 18 support + +We have brought many features specific to Erlang 18. Here are the highlights: + + * Maps can now scale from dozens to millions of keys. Therefore, usage of the modules `Dict` and `HashDict` is now discouraged and will be deprecated in future releases, instead use `Map`. Similarly, `Set` and `HashSet` will be deprecated in favor of `MapSet` + * Compilation times are ~15% faster on average due to improvements in both the Elixir and Erlang compilers and by using maps more extensively in the compilation stack + * Dialyzer now emits less false negative warnings thanks to new annotations available in the Erlang compiler + +## Language improvements + +This release includes many notable language improvements. + +The first of them was the addition of multi aliases/imports/require. Often developers would write: + +```elixir +alias MyApp.Foo +alias MyApp.Bar +alias MyApp.Baz +``` + +Now it can be written in one line by using the new multi syntax: + +```elixir +alias MyApp.{Foo, Bar, Baz} +``` + +We have also added support for variables in map keys. Now you can write: + +```elixir +iex> key = :hello +iex> value = "world" +iex> %{key => value} +%{:hello => "world"} +``` + +Furthermore, variables can also be used on pattern matching along-side the pin operator: + +```elixir +iex> key = :hello +iex> %{^key => value} = %{:hello => "another world"} +iex> value +"another world" +``` + +Finally, Elixir v1.2 introduces the `with` special form that allows developers to match on multiple expressions concisely. Previously, one would write + +```elixir +case File.read("my_file.ex") do + {:ok, contents} -> + case Code.eval_string(contents) do + {res, _binding} -> + {:ok, res} + error -> + error + error -> error + error +end +``` + +such can now be rewritten as + +```elixir +with {:ok, contents} <- File.read("my_file.ex"), + {res, binding} <- Code.eval_string(contents), + do: {:ok, res} +``` + +`with` will match each left side of `<-` against the right side, executing expressions until one of those match fails or until the `do: expression` is performed. In case a match fails, the non-matching result is returned. + +These improvements aim to make the language more consistent and expressive. + +## Getting started experience + +We have also improved both the parser and compiler to be more aware of language constructs, emitting warnings on common pitfalls like when piping to expressions without parentheses or when defining unsafe variables. Such improvements will point developers to the more idiomatic way of writing Elixir code early on. + +Elixir v1.2 also introduces the `i/1` helper in IEx, which allows developers to retrieve information about any data type. This will help newcomers explore the language values while providing experienced developers with crucial information about the value they are introspecting. For example, giving a PID to `i/1` will show if it has a registered name, linked processes and more. Giving it a module, like `i(String)`, shows compile-time information and others. + +All of those improvements tie nicely with our updates to the Getting Started guide, ensuring learning Elixir is more fun and efficient than ever before. + +## Workflow improvements + +One of Elixir goals is to build upon the abstractions provided by Erlang/OTP and make them more productive by focusing on the tooling aspect. + +One of such efforts resulted in "Umbrella Projects", which allows developers to build multiple applications side-by-side, but still run and test them in isolation when desired. Because each application contains its own configuration, supervision tree and initialization cycle, this gives developers the proper mechanisms to break monolithic applications apart without introducing the complexity of managing multiple, different repositories. + +Up to this release, umbrella applications shared mostly dependencies, which meant each application still had their own build directory and their own compilation cycle. Elixir v1.2 allows developers to also share both build and configuration files. This change allows teams to drastically reduce compilation times in umbrella projects by adding the following configuration to each umbrella app's `mix.exs` file: + +```elixir +build_path: "../../_build", +config_path: "../../config/config.exs", +``` + +Umbrella applications generated with Elixir v1.2 will by default include this configuration. The downside of this approach is that applications are a bit less isolated, since configuration is now shared across all projects, although developers can revert back to the previous behaviour by simply removing the flags above. + +Finally, Mix will now consolidate protocols by default as we are now able to consolidate in parallel and cache the consolidation results, providing the best performance across all environments without affecting compilation times. + +These are great additions on top of the faster compilation times we have achieved when migrating to Erlang 18. + +## Rebar 3 support + +With Rebar 3 gaining more adoption in the Erlang community, Mix is now able to fetch and compile Rebar 3 dependencies. This feature is currently experimental and therefore opt-in: if you have a Rebar 3 dependency, you can ask Mix to use Rebar 3 to compile it by passing the `manager: :rebar3` option. Once configured, Mix will prompt you to install Rebar 3 if it is not yet available. + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.2.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Happy coding! diff --git a/_posts/2016-06-21-elixir-v1-3-0-released.markdown b/_posts/2016-06-21-elixir-v1-3-0-released.markdown new file mode 100644 index 000000000..3ee45d0e0 --- /dev/null +++ b/_posts/2016-06-21-elixir-v1-3-0-released.markdown @@ -0,0 +1,288 @@ +--- +layout: post +title: Elixir v1.3 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.3 brings many improvements to the language, the compiler and its tooling, specially Mix (Elixir's build tool) and ExUnit (Elixir's test framework). +--- + +Elixir v1.3 brings many improvements to the language, the compiler and its tooling, specially Mix (Elixir's build tool) and ExUnit (Elixir's test framework). The most notable additions are the new Calendar types, the new cross-reference checker in Mix, and the assertion diffing in ExUnit. We will explore all of them and a couple more enhancements below. + +With this release, we also welcome [Andrea Leopardi](http://github.com/whatyouhide) to Elixir Core Team. He has contributed greatly to this release and maintains important packages in the community, like [Gettext](https://github.com/elixir-lang/gettext) and [Redix](https://github.com/whatyouhide/redix). + +## Language improvements + +The language has been improved semantically and includes new types and APIs. Let's see the three major features. + +### Deprecation of imperative assignment + +Elixir will now warn if constructs like `if`, `case` and friends assign to a variable that is accessed in an outer scope. As an example, imagine a function called `format` that receives a message and some options and it must return a path alongside the message: + +```elixir +def format(message, opts) do + path = + if (file = opts[:file]) && (line = opts[:line]) do + relative = Path.relative_to_cwd(file) + message = Exception.format_file_line(relative, line) <> " " <> message + relative + end + + {path, message} +end +``` + +The `if` block above is implicitly changing the value in `message`. Now imagine we want to move the `if` block to its own function to clean up the implementation: + +```elixir +def format(message, opts) do + path = with_file_and_line(message, opts) + {path, message} +end + +defp with_file_and_line(message, opts) do + if (file = opts[:file]) && (line = opts[:line]) do + relative = Path.relative_to_cwd(file) + message = Exception.format_file_line(relative, line) <> " " <> message + relative + end +end +``` + +The refactored version is broken because the `if` block was actually returning two values, the relative path *and* the new message. Elixir v1.3 will warn on such cases, forcing both variables to be explicitly returned from `if`, `case` and other constructs. Furthermore, this change gives us the opportunity to unify the language scoping rules in future releases. + +### Calendar types and sigils + +Elixir v1.3 introduces the `Calendar` module as well as 4 new calendar types: + + * `Date` - used to store dates (year, month, day) in a given calendar + * `Time` - used to store time (hour, minute, second, microseconds) + * `NaiveDateTime` - used to store datetimes without a timezone (year, month, day, hour, minute, second, microseconds) in a given calendar. It is called naïve because without a timezone, the datetime may not actually exist. For example, when there are daylight savings changes, a whole hour may not exist (when the clock moves forward) or a particular instant may happen twice (when the clock moves backwards) + * `DateTime` - used to store datetimes with timezone (year, month, day, hour, minute, second, microsecond and time zone, with abbreviation, UTC and standard offset) + +The aim of the current implementation of the Calendar modules and its types is to provide a base for interoperatibility in the ecosystem instead of full-featured datetime API. This release includes basic functionality for building new types and converting them from and back strings. + +Elixir v1.3 also introduces 3 new sigils related to the types above: + + * `~D[2016-05-29]` - builds a new date + * `~T[08:00:00]` and `~T[08:00:00.285]` - builds a new time (with different precisions) + * `~N[2016-05-29 08:00:00]` - builds a naive date time + +### Access selectors + +This release introduces new accessors to make it simpler for developers to traverse nested data structures, traversing and updating data in different ways. For instance, given a user with a list of languages, here is how to deeply traverse the map and convert all language names to uppercase: + +```elixir +iex> user = %{name: "john", +...> languages: [%{name: "elixir", type: :functional}, +...> %{name: "c", type: :procedural}]} +iex> update_in user, [:languages, Access.all(), :name], &String.upcase/1 +%{name: "john", + languages: [%{name: "ELIXIR", type: :functional}, + %{name: "C", type: :procedural}]} +``` + +You can see the new accessors in the `Access` module. + +## Mix + +Mix includes new tasks to improve your everyday workflow. Some of those tasks relies on many compiler improvements to know more about your code, providing static analysis to find possible bugs in your code and faster compilation cycles. + +### Compiling n files + +Mix no longer announces every file it compiles. Instead it outputs how many files there is to compile per compilers. Here is the output for a project like [`gettext`](https://github.com/elixir-lang/gettext): + +``` +Compiling 1 file (.yrl) +Compiling 1 file (.erl) +Compiling 19 files (.ex) +Generated gettext app +``` + +In case a file is taking too long to compile, Mix will announce such, for example: + +``` +Compiling lib/gettext.ex (it's taking more than 10s) +``` + +The goal of these changes is to put an increased focus on the "warnings" emitted by the compiler. + +In any case, the previous behaviour can be brought back with the `--verbose` flag and the compilation threshold for files that are taking long can be set via the `--long-compilation-threshold` option. + +### mix xref + +Speaking about warnings, Mix v1.3 includes a new task called `xref` that performs cross reference checks in your code. One of such checks is the ability to find calls to modules and functions that do not exist. For example, if in your library code you call `ThisModuleDoesNotExist.foo(1, 2, 3)`, `mix xref unreachable` will be able to find such code and let you know about it. + +Since such checks can discover possible bugs in your codebase, a new compiler called `xref` has been added to `Mix.compilers/0`, so it runs by default every time you compile your code. [PragTob has written an article exploring how this new compiler has found bugs in existing projects](https://pragtob.wordpress.com/2016/06/02/elixir-1-3s-mix-xref-working-its-magic-in-a-real-world-example/). + +We have included other modes in `xref`, such as: + + * `mix xref callers Foo` - used to find all places in your code that calls a function from the module `Foo` + + * `mix xref graph` - generates a graph with dependencies between source files + +You can find documentation for all modes by running `mix help xref`. We hope tools and text editors can leverage such features to provide useful functionality for their users, helping developers understand code complexity and finding bugs early on. + +### Better dependency tracking + +Besides `xref`, Elixir v1.3 provides better module tracking generally. For example, in previous versions, if you changed a `:path` dependency, Elixir would always fully recompile the current project. In this release, we have improved the tracking algorithms such that, if you change a `:path` dependency, only the files that depend on such dependency are recompiled. + +Such improvements do not only make compilation faster but they also make working with umbrella applications much more productive. Previously, changing a sibling application triggered a full project recompilation, now Elixir can track between sibling applications and recompile only what is needed. + +### mix app.tree and deps.tree + +Mix also includes both `mix app.tree` and `mix deps.tree`. The first will list all applications your current project needs to start in order to boot (i.e. the ones listed in `application/0` in your `mix.exs`) while the second will lists all of your dependencies and so on recursively. + +Here is a quick example from [Plug](https://github.com/elixir-lang/plug): + +```elixir +$ mix app.tree +plug +├── elixir +├── crypto +├── logger +│ └── elixir +└── mime + └── elixir +``` + +The `--format dot` option can also be given to generate graph files to be opened by [GraphViz](http://www.graphviz.org). For example, here is the output of running `mix deps.tree --format dot --only prod` in the [Phoenix web framework](http://phoenixframework.org): + +

+ mix deps.tree for Phoenix in production +

+ +### mix escript.install + +Mix also includes `mix escript.install` and `mix escript.uninstall` tasks for managing escripts. The tasks was designed in a way to mimic the existing `mix archive` functionality except that: + + * Archives must be used sparingly because every new archive installed affects Mix performance, as every new archive is loaded when Mix boots. Escripts solve this by being managed apart from your Elixir/Mix installed + * Archives depends on the current Elixir version. Therefore, updating your Elixir version may break an archive. Fortunately, escripts include Elixir inside themselves, and therefore do not depend on your Elixir system version + +Escripts will be installed at `~/.mix/escripts` which must be added to your [`PATH` environment variable](https://en.wikipedia.org/wiki/PATH_(variable)). + +### Option parser integration + +Elixir v1.3 includes improvements to the option parser, including `OptionParser.parse!/2` and `OptionParser.parse_head!/2` functions that will raise in case of invalid or unknown switches. Mix builds on top of this functionality to provide automatic error reporting solving a common complaint where invalid options were not reported by Mix tasks. + +For example, invoking `mix test --unknown` in earlier Elixir versions would silently discard the `--unknown` option. Now `mix test` correctly reports such errors: + +``` +$ mix test --unknown +** (Mix) Could not invoke task "test": 1 error found! +--unknown : Unknown option +``` + +Note not all tasks have been updated to use strict option parsing. Some tasks, like `mix compile`, are actually a front-end to many other tasks, and as such, it cannot effectively assert which options are valid. + +## ExUnit + +ExUnit packs many improvements on the tooling side, better integration with external tools, as well as mechanisms to improve the readability of your tests. + +### mix test \-\-stale + +ExUnit builds on top of `mix xref` to provide the `mix test --stale` functionality. When the `--stale` flag is given, `mix` will only run the tests that may have changed since the last time you ran `mix test --stale`. For example: + + * If you saved a test file on disk, Mix will run that file and ignore the ones that have not changed + * If you changed a library file, for example, `lib/foo.ex` that defines `Foo`, any test that invokes a function in `Foo` directly or indirectly will also run + * If you modify your `mix.exs` or your `test/test_helper.exs`, Mix will run the whole test suite + +This feature provides a great workflow for developers, allowing them to effortlessly focus on parts of the codebase when developing new features. + +### Diffing + +ExUnit will now include diff-ing output every time a developer asserts `assert left == right` in their tests. For example, the assertion: + +```elixir +assert "fox jumps over the lazy dog" == + "brown fox jumps over the dog" +``` + +will fail with + +![ExUnit diff](/images/contents/exunit-diff.png) + +such that "lazy" in "lhs" will be shown in red to denote it has been removed from "rhs" while "brown" in "rhs" will be shown in green to denote it has been added to the "rhs". + +When working with large or nested data structures, the diffing algorithm makes it fast and convenient to spot the actual differences in the asserted values. + +### Test types + +ExUnit v1.3 includes the ability to register different test types. This means libraries like QuickCheck can now provide functionality such as: + +```elixir +defmodule StringTest do + use ExUnit.Case, async: true + use PropertyTestingLibrary + + property "starts_with?" do + forall({s1, s2} <- {utf8, utf8}) do + String.starts_with?(s1 <> s2, s1) + end + end +end +``` + +At the end of the run, ExUnit will also report it as a property, including both the amount of tests and properties: + +``` +1 property, 10 tests, 0 failures +``` + +### Named setups and describes + +Finally, ExUnit v1.3 includes the ability to organize tests together in describe blocks: + +```elixir +defmodule StringTest do + use ExUnit.Case, async: true + + describe "String.capitalize/2" do + test "uppercases the first grapheme" do + assert "T" <> _ = String.capitalize("test") + end + + test "lowercases the remaining graphemes" do + assert "Test" = String.capitalize("TEST") + end + end +end +``` + +Every test inside a describe block will be tagged with the describe block name. This allows developers to run tests that belong to particular blocks, be them in the same file or across many files: + +``` +$ mix test --only describe:"String.capitalize/2" +``` + +Note describe blocks cannot be nested. Instead of relying on hierarchy for composition, we want developers to build on top of named setups. For example: + +```elixir +defmodule UserManagementTest do + use ExUnit.Case, async: true + + describe "when user is logged in and is an admin" do + setup [:log_user_in, :set_type_to_admin] + + test ... + end + + describe "when user is logged in and is a manager" do + setup [:log_user_in, :set_type_to_manager] + + test ... + end + + defp log_user_in(context) do + # ... + end +end +``` + +By restricting hierarchies in favor of named setups, it is straight-forward for the developer to glance at each describe block and know exactly the setup steps involved. + +## Summing up + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.3.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Happy coding! diff --git a/_posts/2016-07-14-announcing-genstage.markdown b/_posts/2016-07-14-announcing-genstage.markdown new file mode 100644 index 000000000..7a853d043 --- /dev/null +++ b/_posts/2016-07-14-announcing-genstage.markdown @@ -0,0 +1,362 @@ +--- +layout: post +title: Announcing GenStage +authors: +- José Valim +category: Announcements +excerpt: GenStage is a new Elixir behaviour for exchanging events with back-pressure between Elixir processes. In this blog post we will cover the background that led us to GenStage, some example use cases, and what we are exploring for future releases. +--- + +Today we are glad to announce the official release of GenStage. GenStage is a new Elixir behaviour for exchanging events with back-pressure between Elixir processes. In the short-term, we expect GenStage to replace the use cases for GenEvent as well as providing a composable abstraction for consuming data from third-party systems. + +In this blog post we will cover the background that led us to GenStage, some example use cases, and what we are exploring for future releases. If instead you are looking for a quick reference, [check the project source code](https://github.com/elixir-lang/gen_stage) and [access its documentation](https://hexdocs.pm/gen_stage/Experimental.GenStage.html). + +## Background + +One of the original motivations for [creating and designing Elixir was to introduce better abstractions for working with collections](https://www.youtube.com/watch?v=Lqo9-pQuRKE). Not only that, we want to provide developers interested in manipulating collections with a path to take their code from eager to lazy, to concurrent and then distributed. + +Let's discuss a simple but actual example: word counting. The idea of word counting is to receive one file and count how many times each word appears in the document. Using the `Enum` module it could be implemented as follows: + +```elixir +File.read!("path/to/some/file") +|> String.split("\n") +|> Enum.flat_map(fn line -> + String.split(line, " ") + end) +|> Enum.reduce(%{}, fn word, acc -> + Map.update(acc, word, 1, & &1 + 1) + end) +|> Enum.to_list() +``` + +While the solution above works fine and is efficient for small files, it is quite restrictive for large inputs as it loads the whole file into memory. + +Another issue with the solution above is that the `Enum.flat_map/2` step will build a huge list, with all the words in the file, before we effectively start counting them. Again, for a large document, this means more memory usage and a waste of processing time in building a list that will be traversed right after. + +Luckily, Elixir provides a solution to this problem (and has provided it for quite some time): streams. One of the advantage of streams is they are lazy, allowing us to traverse collections item by item, in this case, line by line, instead of loading the whole data set into memory. Let's rewrite the example above to use streams: + +```elixir +File.stream!("path/to/some/file") +|> Stream.flat_map(fn line -> + String.split(line, " ") + end) +|> Enum.reduce(%{}, fn word, acc -> + Map.update(acc, word, 1, & &1 + 1) + end) +|> Enum.to_list() +``` + +By using `File.stream!` and `Stream.flat_map`, we build a lazy computation that will emit a single line, break that line into words, and emit such words one by one without building huge lists in memory when enumerated. The functions in the [Stream module](https://hexdocs.pm/elixir/Stream.html) just express the computation we want to perform. The computation itself, like traversing the file or breaking into words in `flat_map`, only happens when we call a function in the `Enum` module. We have covered [the foundation for Enum and Streams](https://dashbit.co/blog/introducing-reducees) in another article. + +The solution above allows us to work with large datasets without loading them all into memory. For large files, it is going to provide much better performance than the eager version. However, the solution above still does not leverage concurrency. For a machine with more than one core, which is the huge majority of machines we have available today, it is a suboptimal solution. + +That said, how could we leverage concurrency in the example above? + +During my ElixirConf 2015 keynote, [I discussed one of the most immediate solutions to this problem](http://confreaks.tv/videos/elixirconf2015-keynote) which was to convert parts of your pipeline to separate processes: + +```elixir +File.stream!("path/to/some/file") +|> Stream.flat_map(fn line -> + String.split(line, " ") + end) +|> Stream.async() # NEW! +|> Enum.reduce(%{}, fn word, acc -> + Map.update(acc, word, 1, & &1 + 1) + end) +|> Enum.to_list() +``` + +The idea is that `Stream.async` would run the previous computations in a separate process that would stream its messages to the process that called `Enum.reduce`. Unfortunately, the solution above is less than ideal. + +First of all, we want to avoid moving data between processes as much as possible. Instead, we want to start multiple processes that perform the same computation in parallel. Not only that, if we are requiring developers to place `Stream.async` manually, it may lead to inefficient and error prone solutions. + +Although the solution above has many flaws, it has helped us ask the right questions: + + * If `Stream.async` is introducing new processes, how can we guarantee those processes are supervised? + + * Since we are exchanging messages between processes, how do we prevent a process from receiving too many messages? We need a back-pressure mechanism that allows the receiving process to specify how much it can handle from the sending process. + +We have jumped through different abstractions trying to answer those questions until we have finally settled on GenStage. + +## GenStage + +GenStage is a new Elixir behaviour for exchanging events with back-pressure between Elixir processes. Developers who use GenStage only need to worry about how the data is produced, manipulated and consumed. The act of dispatching the data and providing back-pressure is completely abstracted away from the developers. + +As a quick example, let's write a simple pipeline that will produce events as increasing numbers, multiply those numbers by two, and then print them to the terminal. We will do so by implementing three stages, the `:producer`, the `:producer_consumer` and the `:consumer`, which we will call `A`, `B` and `C` respectively. We will go back to the word counting example at the end of this post. + +Let's start with the producer that we will call `A`. Since `A` is a producer, its main responsibility is to receive demand, which is the number of events the consumer is willing to handle, and generate events. Those events may be in memory or an external data source. For now let's implement a simple counter starting from a given value of `counter` received on `init/1`: + +Note: all of the modules in the `GenStage` project are prefixed with the `Experimental` namespace. That's why the examples below and your code should `alias Experimental.GenStage` at the top of your files. + +```elixir +alias Experimental.GenStage + +defmodule A do + use GenStage + + def init(counter) do + {:producer, counter} + end + + def handle_demand(demand, counter) when demand > 0 do + # If the counter is 3 and we ask for 2 items, we will + # emit the items 3 and 4, and set the state to 5. + events = Enum.to_list(counter..counter+demand-1) + + # The events to emit is the second element of the tuple, + # the third being the state. + {:noreply, events, counter + demand} + end +end +``` + +`B` is a producer-consumer. This means it does not explicitly handle the demand because the demand is always forwarded to its producers. Once `A` receives the demand from `B`, it will send events to `B` which will be transformed by `B` as desired and then sent to `C`. In our case, B will receive events and multiply them by a number given on initialization and stored as the state: + +```elixir +alias Experimental.GenStage + +defmodule B do + use GenStage + + def init(number) do + {:producer_consumer, number} + end + + def handle_events(events, _from, number) do + events = Enum.map(events, & &1 * number) + {:noreply, events, number} + end +end +``` + +`C` is the consumer which will finally receive those events and print them every second to the terminal: + +```elixir +alias Experimental.GenStage + +defmodule C do + use GenStage + + def init(sleeping_time) do + {:consumer, sleeping_time} + end + + def handle_events(events, _from, sleeping_time) do + # Print events to terminal. + IO.inspect(events) + + # Sleep the configured time. + Process.sleep(sleeping_time) + + # We are a consumer, so we never emit events. + {:noreply, [], sleeping_time} + end +end +``` + +With the stages defined, we can start and connect them: + +```elixir +{:ok, a} = GenStage.start_link(A, 0) # starting from zero +{:ok, b} = GenStage.start_link(B, 2) # multiply by 2 +{:ok, c} = GenStage.start_link(C, 1000) # sleep for a second + +GenStage.sync_subscribe(c, to: b) +GenStage.sync_subscribe(b, to: a) + +# Sleep so we see events printed. +Process.sleep(:infinity) +``` + +As soon as we subscribe the stages, we should see items being printed to the terminal. Notice that, even though we have introduced a sleep command to the consumer, the producers will never overflow the consumer with data. That's because the communication between stages is demand-driven. The producer can only send items to consumers after the consumers have sent demand upstream. The producer must never send more items than the consumer has specified. + +One consequence of this design decision is that parallelizing stateless stages like the consumer above is really straightforward: + +```elixir +{:ok, a} = GenStage.start_link(A, 0) # starting from zero +{:ok, b} = GenStage.start_link(B, 2) # multiply by 2 + +{:ok, c1} = GenStage.start_link(C, 1000) # sleep for a second +{:ok, c2} = GenStage.start_link(C, 1000) # sleep for a second +{:ok, c3} = GenStage.start_link(C, 1000) # sleep for a second +{:ok, c4} = GenStage.start_link(C, 1000) # sleep for a second + +GenStage.sync_subscribe(c1, to: b) +GenStage.sync_subscribe(c2, to: b) +GenStage.sync_subscribe(c3, to: b) +GenStage.sync_subscribe(c4, to: b) +GenStage.sync_subscribe(b, to: a) + +# Sleep so we see events printed. +Process.sleep(:infinity) +``` + +By simply starting multiple consumers, the stage `B` will now receive demand from multiple stages and dispatch events to those stages which are now running concurrently, always picking the stage that is able to process more items. We can also leverage concurrency from the opposite direction: if the producer is the slow stage in a pipeline, you can start multiple producers and have each consumer subscribe to them. + +In order to know which consumer should receive a particular event, producer stages depend on a behaviour called [`GenStage.Dispatcher`](https://hexdocs.pm/gen_stage/Experimental.GenStage.Dispatcher.html). The default dispatcher is the `GenStage.DemandDispatcher` we have briefly described above: it will collect the demand from different consumers and dispatch to the one with highest demand. This means if one consumer is slow, maybe because we increased its sleeping time to 10 seconds, it will receive less items. + +### GenStage for data-ingestion + +One of the use cases for GenStage is to consume data from third-party systems. The demand system with back-pressure guarantees we won't import more data than we can effectively handle. The demand dispatcher allows us to easily leverage concurrency when processing the data by simply adding more consumers. + +During the Elixir London Meetup, I have live-coded a short example that shows how to use `GenStage` to concurrently process data stored in a PostgreSQL database as a queue: + + + +### GenStage for event dispatching + +Another scenario where GenStage can be useful today is to replace cases where developers would have used [GenEvent](https://hexdocs.pm/elixir/GenEvent.html) in the past. For those unfamiliar with GenEvent, it is a behaviour where events are sent to an "event manager" which then proceeds to invoke "event handlers" for each event. GenEvent, however, has one big flaw: the event manager and all event handlers run in the same process. This means GenEvent handlers cannot easily leverage concurrency without forcing developers to implement those mechanisms themselves. Furthermore, GenEvent handlers have very awkward error semantics. Because event handlers are not separate processes, we cannot simply rely on supervisors restarting them. + +GenStage solves those problems by having a producer as the event manager. The producer itself should be configured to use [`GenStage.BroadcastDispatcher`](https://hexdocs.pm/gen_stage/Experimental.GenStage.BroadcastDispatcher.html) as its dispatcher. The broadcast dispatcher will guarantee events are dispatched to all consumers in a way that does not exceed the demand of any of the consumers. This allows us to leverage concurrency and having the "event manager" as a producer gives us much more flexibility in terms of buffering and reacting to failures. + +Let's see an example of building an event manager as a producer: + +```elixir +alias Experimental.GenStage + +defmodule EventManager do + use GenStage + + @doc """ + Starts the manager. + """ + def start_link() do + GenStage.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @doc """ + Sends an event and returns only after the event is dispatched. + """ + def sync_notify(event, timeout \\ 5000) do + GenStage.call(__MODULE__, {:notify, event}, timeout) + end + + ## Callbacks + + def init(:ok) do + {:producer, {:queue.new, 0}, dispatcher: GenStage.BroadcastDispatcher} + end + + def handle_call({:notify, event}, from, {queue, demand}) do + dispatch_events(:queue.in({from, event}, queue), demand, []) + end + + def handle_demand(incoming_demand, {queue, demand}) do + dispatch_events(queue, incoming_demand + demand, []) + end + + defp dispatch_events(queue, demand, events) do + with d when d > 0 <- demand, + {item, queue} = :queue.out(queue), + {:value, {from, event}} <- item do + GenStage.reply(from, :ok) + dispatch_events(queue, demand - 1, [event | events]) + else + _ -> {:noreply, Enum.reverse(events), {queue, demand}} + end + end +end +``` + +The `EventManager` works as a buffer. If there is demand but not events to be sent, we store such demand. If there are events but no demand, we store such events in a queue. If a client tries to broadcast an event, the `sync_notify` call will block until the event is effectively broadcasted. The bulk of the logic is in the `dispatch_events/3` function that takes events from the queue while there is demand. + +By implementing the event manager as a producer, we can configure all sorts of behaviours that are simply not possible with `GenEvent`, such as how much data we want to queue (or for how long) and if events should be buffered or not when there are no consumers (via the `handle_subscribe/4` and `handle_cancel/3` callbacks). + +Implementing event handlers is as straightforward as writing any other consumer. We could in fact use the `C` consumer implemented earlier. However, given event managers are often defined before the handlers, it is recommended for handlers to subscribe to managers when they start: + +```elixir +alias Experimental.GenStage + +defmodule EventHandler do + use GenStage + + def start_link() do + GenStage.start_link(__MODULE__, :ok) + end + + # Callbacks + + def init(:ok) do + # Starts a permanent subscription to the broadcaster + # which will automatically start requesting items. + {:consumer, :ok, subscribe_to: [EventManager]} + end + + def handle_events(events, _from, state) do + IO.inspect events + {:noreply, [], state} + end +end +``` + +Such guarantees that, if a supervised `EventHandler` crashes, the supervisor will start a new event handler which will promptly subscribe to the same manager, solving the awkward error handling semantics we have seen with `GenEvent`. + +## The path forward + +With the release of GenStage v0.3.0, we have reached an important milestone as `GenStage` can be used as both event managers and a way to exchange events between processes, often external data sources, with back-pressure. + +The v0.3.0 release also includes the [`GenStage.stream`](https://hexdocs.pm/gen_stage/Experimental.GenStage.html#stream/1) function, which allows us to consume data from a GenStage as a stream, and [`GenStage.from_enumerable`](https://hexdocs.pm/gen_stage/Experimental.GenStage.html#from_enumerable/2) which allows us to use an enumerable or a stream, like `File.stream!`, as a producer. Closing the gap between stages and streams. + +However, we are far from done! + +First of all, now is the moment for the community to step in and try GenStage out. If you have used GenEvent in the past, can it be replaced by a GenStage? Similarly, if you were planning to implement an event handling system, give GenStage a try. + +Developers who maintain libraries that integrate with external data sources, be it a RabbitMQ, Redis or Apacha Kafka, can explore GenStage as an abstraction for consuming data from those sources. Library developers must implement producers and leave it up for their users to configure the consumer stages. + +Once we get enough feedback, `GenStage` will be included in some shape as part of the standard library. The goal is to introduce `GenStage` and phase `GenEvent` out in the long term. + +We, on the Elixir team, have just got started too. The next milestone for GenStage is to revisit the original problem and provide developers a clear path to take their collection processing code from eager, to lazy, to concurrent (and then distributed). + +As seen earlier, today we allow developers to transform eager code into lazy by introducing streams. + +```elixir +File.stream!("path/to/some/file") +|> Stream.flat_map(fn line -> + String.split(line, " ") + end) +|> Enum.reduce(%{}, fn word, acc -> + Map.update(acc, word, 1, & &1 + 1) + end) +|> Enum.to_list() +``` + +While the above is helpful when working with large or infinite collections, it still does not leverage concurrency. To address that, we are currently exploring a solution named [`GenStage.Flow`](https://hexdocs.pm/gen_stage/Experimental.Flow.html), that allows us to express our computations similarly to streams, except they will run across multiple stages instead of a single process: + +```elixir +alias Experimental.GenStage.Flow +File.stream!("path/to/some/file") +|> Flow.from_enumerable() +|> Flow.flat_map(fn line -> + for word <- String.split(" "), do: {word, 1} + end) +|> Flow.reduce_by_key(& &1 + &2) +|> Enum.to_list() +``` + +And the highly optimized version: + +```elixir +alias Experimental.GenStage.Flow + +# Let's compile common patterns for performance +empty_space = :binary.compile_pattern(" ") # NEW! + +File.stream!("path/to/some/file", read_ahead: 100_000) # NEW! +|> Flow.from_enumerable() +|> Flow.flat_map(fn line -> + for word <- String.split(empty_space), do: {word, 1} + end) +|> Flow.partition_with(storage: :ets) # NEW! +|> Flow.reduce_by_key(& &1 + &2) +|> Enum.to_list() +``` + +Flow will look at the computations we want to perform and start a series of stages to execute our code while keeping the amount of data being transferred between processes to a minimum. If you are interested in `GenStage.Flow` and how the computations above are spread across multiple stages, [we have written some documentation based on the prototypes we have built so far](https://hexdocs.pm/gen_stage/Experimental.Flow.html). The code itself is coming in future GenStage releases. We will also have to consider how the `GenStage.Flow` API mirrors the functions in `Enum` and `Stream` to make the path from eager to concurrent clearer. + +For the word counting problem with a fixed data, early experiments show a linear increase in performance with a fixed overhead of 20%. In other words, a dataset that takes 60s with a single core, takes 36s on a machine with 2 cores and 18s in one with 4 cores. All of those gains by simply moving your computations from streams to Flow. We plan to benchmark on machines with over 40 cores soon. + +We are very excited with the possibilities GenStage brings to developers and all new paths it allows us to explore and research. So give it a try and let us know! [GenStage, Flows, and more will also be the topic of my keynote at ElixirConf 2016](http://www.elixirconf.com/) and we hope to see you there. + +Finally, we want to thank the [akka-streams and reactive-streams projects](http://reactive-streams.io) which provided us guidance in implementing the demand-driven exchange between stages as well as the [Apache Spark](http://spark.apache.org/) and [Apache Beam](http://beam.incubator.apache.org/) initiatives that inspire the work behind `GenStage.Flow`. + +Happy coding! diff --git a/_posts/2017-01-05-elixir-v1-4-0-released.markdown b/_posts/2017-01-05-elixir-v1-4-0-released.markdown new file mode 100644 index 000000000..f110bb07e --- /dev/null +++ b/_posts/2017-01-05-elixir-v1-4-0-released.markdown @@ -0,0 +1,135 @@ +--- +layout: post +title: Elixir v1.4 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.4 brings many improvements to the language, its standard library and the Mix build tool. +--- + +Elixir v1.4 brings new features, enhancements and bug fixes. The most notable changes are the addition of the `Registry` module, the `Task.async_stream/3` and `Task.async_stream/5` function which aid developers in writing concurrent software, and the new application inference and commands added to Mix. + +In this post we will cover the main additions. The complete [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.4.0) are also available. + +## Registry + +The [`Registry`](https://hexdocs.pm/elixir/Registry.html) is a new module in Elixir's standard library that allows Elixir developers to implement patterns such as name lookups, code dispatching or even a pubsub system in a simple and scalable way. + +Broadly speaking, the Registry is a local, decentralized and scalable key-value process storage. Let's break this in parts: + + * Local because keys and values are only accessible to the current node (opposite to distributed) + * Decentralized because there is no single entity responsible for managing the registry + * Scalable because performance scales linearly with the addition of more cores upon partitioning + +A registry may have unique or duplicate keys. Every key-value pair is associated to the process registering the key. Keys are automatically removed once the owner process terminates. Starting, registering and looking up keys is quite straight-forward: + +```elixir +iex> Registry.start_link(:unique, MyRegistry) +iex> {:ok, _} = Registry.register(MyRegistry, "hello", 1) +iex> Registry.lookup(MyRegistry, "hello") +[{self(), 1}] +``` + +Finally, huge thanks to [Bram Verburg](https://twitter.com/voltonez) who has performed [extensive benchmarks](https://docs.google.com/spreadsheets/d/1MByRZJMCnZ1wPiLhBEnSRRSuy1QXp8kr27PIOXO3qqg/edit#gid=0) on the registry to show it scales linearly with the number of cores by increasing the number of partitions. + +## Syntax coloring + +Elixir v1.4 introduces the ability to syntax color inspected data structures and IEx automatically relies on this feature to provide syntax coloring for evaluated shell results: + +![IEx coloring](/images/contents/iex-coloring.png) + +This behaviour can be configured via the `:syntax_colors` coloring option: + +```elixir +IEx.configure [colors: [syntax_colors: [atom: :cyan, string: :green]]] +``` + +To disable coloring altogether, simply pass an empty list to `:syntax_colors`. + +## Task.async_stream + +When there is a need to traverse a collection of items concurrently, Elixir developers often resort to tasks: + +```elixir +collection +|> Enum.map(&Task.async(SomeMod, :function, [&1])) +|> Enum.map(&Task.await/1) +``` + +The snippet above will spawn a new task by invoking `SomeMod.function(element)` for every element in the collection and then await for the task results. + +However, the snippet above will spawn and run concurrently as many tasks as there are items in the collection. While this may be fine in many occasions, including small collections, sometimes it is necessary to restrict amount of tasks running concurrently, specially when shared resources are involved. + +Elixir v1.4 adds `Task.async_stream/3` and `Task.async_stream/5` which brings some of the lessons we learned from [the GenStage project](/blog/2016/07/14/announcing-genstage/) directly into Elixir: + +```elixir +collection +|> Task.async_stream(SomeMod, :function, [], max_concurrency: 8) +|> Enum.to_list() +``` + +The code above will also start the same `SomeMod.function(element)` task for every element in the collection except it will also guarantee we have at most 8 tasks being processed at the same time. You can use `System.schedulers_online` to retrieve the number of cores and balance the processing based on the amount of cores available. + +The `Task.async_stream` functions are also lazy, allowing developers to partially consume the stream until a condition is reached. Furthermore, `Task.Supervisor.async_stream/4` and `Task.Supervisor.async_stream/6` can be used to ensure the concurrent tasks are spawned under a given supervisor. + +## Application inference + +In previous Mix versions, most of your dependencies had to be added both to your dependencies list and applications list. Here is how a `mix.exs` would look like: + +```elixir +def application do + [applications: [:logger, :plug, :postgrex]] +end + +def deps do + [{:plug, "~> 1.2"}, + {:postgrex, "~> 1.0"}] +end +``` + +This was a common source of confusion and quite error prone as many developers would not list their dependencies in the applications list. + +Mix v1.4 now automatically infers your applications list as long as you leave the `:applications` key empty. The `mix.exs` above can be rewritten to: + +```elixir +def application do + [extra_applications: [:logger]] +end + +def deps do + [{:plug, "~> 1.2"}, + {:postgrex, "~> 1.0"}] +end +``` + +With the above, Mix will automatically build your application list based on your dependencies. Developers now only need to specify which applications shipped as part of Erlang or Elixir that they require, such as `:logger`. + +Finally, if there is a dependency you don't want to include in the application runtime list, you can do so by specifying the `runtime: false` option: + + {:distillery, "> 0.0.0", runtime: false} + +We hope this feature provides a more streamlined workflow for developers who are building releases for their Elixir projects. + +## Mix install from SCM + +Mix v1.4 can now install escripts and archives from both Git and Hex, providing you with even more options for distributing Elixir code. + +This makes it possible to distribute CLI applications written in Elixir by publishing a package which builds an escript to Hex. [`ex_doc`](https://hex.pm/packages/ex_doc) has been updated to serve as an example of how to use this new functionality. + +Simply running: + + mix escript.install hex ex_doc + +will fetch `ex_doc` and its dependencies, build them, and then install `ex_doc` to `~/.mix/escripts` (by default). After adding `~/.mix/escripts` to your `PATH`, running `ex_doc` is as simple as: + + ex_doc + +You can now also install archives from Hex in this way. Since they are fetched and built on the user's machine, they do not have the same limitations as pre-built archives. However, keep in mind archives are loaded on every Mix command and may conflict with modules or dependencies in your projects. For this reason, escripts is the preferred format for sharing executables. + +It is also possible to install escripts and archives by providing a Git/GitHub repo. See `mix help escript.install` and `mix help archive.install` for more details. + +## Summing up + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.4.0). Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Happy coding! diff --git a/_posts/2017-07-25-elixir-v1-5-0-released.markdown b/_posts/2017-07-25-elixir-v1-5-0-released.markdown new file mode 100644 index 000000000..c4f38f788 --- /dev/null +++ b/_posts/2017-07-25-elixir-v1-5-0-released.markdown @@ -0,0 +1,179 @@ +--- +layout: post +title: Elixir v1.5 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.5 integrates with Erlang/OTP 20 and bring changes that improves the language reach and the developer experience +--- + +Elixir v1.5 includes new features, enhancements, and bug fixes. While [Elixir v1.4](/blog/2017/01/05/elixir-v1-4-0-released/) focused on tools for concurrency and scalability, Elixir v1.5 brings many improvements to the developer experience and quality of life. As we will see, many of those are powered by the latest Erlang/OTP 20. This is also the last Elixir release that supports Erlang/OTP 18. + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, noscript tags with the proper links will be shown. + +## UTF-8 atoms, function names and variables + +Elixir v1.5 supports non-quoted atoms and variables to be in UTF-8 when using Erlang/OTP 20. For example: + +```elixir +test "こんにちは世界" do + assert :こんにちは世界 +end +``` + +Or: + +```elixir +saudação = "Bom dia!" +``` + +Elixir follows the recommendations in [Unicode Annex #31](http://unicode.org/reports/tr31/) to make Elixir more accessible to other languages and communities. Identifiers must still be a sequence of letters, followed by digits and combining marks. This means symbols, such as mathematical notations and emoji, are not allowed in identifiers. + +For a complete reference on Elixir syntax, see the [Syntax Reference](https://hexdocs.pm/elixir/1.5/syntax-reference.html). For technical details on Unicode support, see [Unicode Syntax](https://hexdocs.pm/elixir/1.5/unicode-syntax.html). + +## IEx helpers and breakpoints + +IEx got many enhancements to the developer experience. + +First of all, the autocompletion system is now capable of autocompleting variables and user imports: + + + +IEx also got new functions, such as `exports/1`, for listing all functions and macros in a module, and the new `runtime_info/0`: + + + +Finally, IEx also features a breakpoint system for code debugging when running on Erlang/OTP 20. The following functions have been added to aid debugging: + + * `break!/2` - sets up a breakpoint for a given `Mod.fun/arity` + * `break!/4` - sets up a breakpoint for the given module, function, arity + * `breaks/0` - prints all breakpoints and their ids + * `continue/0` - continues until the next breakpoint in the same process + * `open/0` - opens editor on the current breakpoint + * `remove_breaks/0` - removes all breakpoints in all modules + * `remove_breaks/1` - removes all breakpoints in a given module + * `reset_break/1` - sets the number of stops on the given id to zero + * `reset_break/3` - sets the number of stops on the given module, function, arity to zero + * `respawn/0` - starts a new shell (breakpoints will ask for permission once more) + * `whereami/1` - shows the current location + +Let's see an example: + + + +In the snippet above we set a breakpoint in the `URI.decode_query/2` function, which is then triggered when invoked the function. We used `whereami/1` to get more information about the surrounded code and we were also able to access the variables at place of debugging. From there, we can either set more breakpoints, remove existing breakpoints and continue execution. The session ended by calling `open`, which will open your editor at the file and line under debugging. `open/1` can also be invoked by passing any module or function, and IEx will open your editor at that place. + +The debugging functions improve the experience both within IEx and during testing. For example, if you are debugging a Phoenix application, you can start `IEx` while running your test suite with `iex -S mix test --trace` and then call `IEx.break!(MyAppWeb.UserController.index/2)` to debug the `index` action of the `UserController`. Note we gave the `--trace` flag to `mix test`, which ensures only one test runs at a time and removes any timeouts from the suite. + +## Exception.blame + +`Exception.blame/3` is a new function in Elixir that is capable of attaching debug information to certain exceptions. Currently this is used to augment `FunctionClauseError`s with a summary of all clauses and which parts of clause match and which ones didn't. Let's try it out: + + + +In the example above, an argument that did not match or guard that did not evaluate to true are shown between in red. If the terminal does not support ANSI coloring, they are wrapped in `-` instead of shown in red. + +Since blaming an exception can be expensive, `Exception.blame/3` must be used exclusively in debugging situations. It is not advised to apply it to production components such as a Logger. This feature has been integrated into the compiler, the command line, ExUnit and IEx. + +This feature also requires Erlang/OTP 20. + +## Streamlined child specs + +Elixir v1.5 streamlines how supervisors are defined and used in Elixir. Elixir now allows child specifications, which specify how a child process is supervised, to be defined in modules. In previous versions, a project using Phoenix would write: + +```elixir +import Supervisor.Spec + +children = [ + supervisor(MyApp.Repo, []), + supervisor(MyApp.Endpoint, []) +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +In Elixir v1.5, one might do: + +```elixir +children = [ + MyApp.Repo, + MyApp.Endpoint +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +The above works by calling the `child_spec/1` function on the given modules. + +This new approach allows `MyApp.Repo` and `MyApp.Endpoint` to control how they run under a supervisor. This reduces the chances of mistakes being made, such as starting an Ecto repository as a worker or forgetting to declare that tasks are temporary in a supervision tree. + +If it is necessary to configure any of the children, such can be done by passing a tuple instead of an atom: + +```elixir +children = [ + {MyApp.Repo, url: "ecto://localhost:4567/my_dev"}, + MyApp.Endpoint +] +``` + +The modules `Agent`, `Registry`, `Task`, and `Task.Supervisor` have been updated to include a `child_spec/1` function, allowing them to be used directly in a supervision tree similar to the examples above. `use Agent`, `use GenServer`, `use Supervisor`, and `use Task` have also been updated to automatically define an overridable `child_spec/1` function. + +Finally, child specifications are now provided as maps (data-structures) instead of the previous `Supervisor.Spec.worker/3` and `Supervisor.Spec.supervisor/3` APIs. This behaviour also aligns with how supervisors are configured in Erlang/OTP 18+. See the updated [`Supervisor`](https://hexdocs.pm/elixir/1.5/Supervisor.html) docs for more information, as well as the new `Supervisor.init/2` and `Supervisor.child_spec/2` functions. + +## @impl + +This release also allows developers to mark which functions in a given module are an implementation of a callback. For example, when using the [Plug](https://github.com/elixir-lang/plug) project, one needs to implement both `init/1` and `call/2` when writing a Plug: + +```elixir +defmodule MyApp do + @behaviour Plug + + def init(_opts) do + opts + end + + def call(conn, _opts) do + Plug.Conn.send_resp(conn, 200, "hello world") + end +end +``` + +The problem with the approach above is that, once more and more functions are added to the `MyApp` module, it becomes increasingly harder to know the purposes of the `init/1` and `call/2` functions. For example, for a developer unfamiliar with Plug, are those functions part of the `MyApp` API or are they implementations of a given callback? + +Elixir v1.5 introduces the `@impl` attribute, which allows us to mark that certain functions are implementation of callbacks: + +```elixir +defmodule MyApp do + @behaviour Plug + + @impl true + def init(_opts) do + opts + end + + @impl true + def call(conn, _opts) do + Plug.Conn.send_resp(conn, 200, "hello world") + end +end +``` + +You may even use `@impl Plug` if you want to explicitly document which behaviour defines the callback you are implementing. + +Overall, using `@impl` has the following advantages: + + * Readability of the code is increased, as it is now clear which functions are part of your API and which ones are callback implementations. To reinforce this idea, `@impl true` automatically marks the function as `@doc false`, disabling documentation unless `@doc` is explicitly set + + * If you define `@impl` before a function that is not a callback, Elixir will error. This is useful in case of typos or in case the behaviour definition changes (such as a new major version of a library you depend on is released) + + * If you use `@impl` in one implementation, Elixir will force you to declare `@impl` for all other implementations in the same module, keeping your modules consistent + +## Calendar improvements + +[Elixir v1.3](https://elixir-lang.org/blog/2016/06/21/elixir-v1-3-0-released/) introduced the Calendar module with the underlying `Time`, `Date`, `NaiveDateTime` and `Datetime` data types. We are glad to announce we consider the base Calendar API to be finished in Elixir v1.5. This release includes many enhancements, such as `Date.range/2` and the ability to convert between different calendars. + +## Summing up + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.5.0). There are many other exciting changes, such as compiler enhancements that reduces compilation times by 10%-15% on averages. When taken into account with the compiler improvements in Erlang/OTP 20 itself, some applications have seen gains up to 30% in compilation times. + +Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. diff --git a/_posts/2017-10-31-stream-data-property-based-testing-and-data-generation-for-elixir.markdown b/_posts/2017-10-31-stream-data-property-based-testing-and-data-generation-for-elixir.markdown new file mode 100644 index 000000000..242c28635 --- /dev/null +++ b/_posts/2017-10-31-stream-data-property-based-testing-and-data-generation-for-elixir.markdown @@ -0,0 +1,184 @@ +--- +layout: post +title: "StreamData: Property-based testing and data generation" +authors: +- Andrea Leopardi +category: Announcements +excerpt: We are working on data generation and property-based testing for the next versions of Elixir. +--- + +In this blog post, we'll talk about property-based testing and sample data generation. We'll cover what these are, why we want them in Elixir, and what are are plans for the future. If you want to use the features discussed here or you want to read more formal documentation, head over to [stream_data][], which is a library that currently provides both features (albeit in beta form) and which is where we are focusing our efforts. + +## Sample data generation + +The core of the [stream_data][] library is `StreamData`: this module provides all the functionalities related to generating sample data of many kinds. It includes both data generators for data types (like integers or booleans) as well as tools to combine other generators (such as `one_of(list_of_generators)`). + +Developers are not supposed to create generators from scratch, but use the provided generators and the provided combinator functions to compose them. An example of a generator is the one returned by `StreamData.integer()`: this function returns a generator that generates integers. Generators are infinite streams of terms that implement the `Enumerable` protocol. This means we can do operations such as taking terms out of a generator through functions from `Enum` and `Stream`: + +```elixir +Enum.take(StreamData.integer(), 5) +#=> [1, -1, 3, 4, 3] +``` + +`StreamData` contains some functions to modify generators. For example, you can build a generator of positive integers on top of `StreamData.integer()` and `StreamData.map/2`: + +```elixir +generator = StreamData.map(StreamData.integer(), &abs/1) +Enum.take(generator, 5) +#=> [0, 1, 3, 3, 2] +``` + +`StreamData.map/2` is encouraged over `Stream.map/2` because generators return values that can shrink, which is something property-based testing takes advantage of as we'll see later on. When treated as enumerables, generators return normal values that cannot be shrunk. + +We decided to separate data-generation from property-based testing because it's something that developers can take advantage of in situations outside of property-based testing. For example, data streams can be used to seed a database or to have randomly generated data available during regular tests. + +## Property-based testing + +We often write tests like this: + +```elixir +test "length/1 calculates the length of a list" do + assert length([]) == 0 + assert length([:one]) == 1 + assert length([1, 2, 3]) == 3 +end +``` + +This test is written using an *example-based approach*. We are writing both the input to the piece of software we are testing as well as the expected output, and the testing tool is verifying that running the software on the given input results in the expected output. This style of testing is common and useful because it lets you get up and running easily and also lets you test known corner cases in an explicit way. However, it's hard to test many cases this way and even harder to uncover *unknown* corner cases that may reveal bugs in your code. + +Property-based testing is an intuitive way to fix some of the problems mentioned above. + +```elixir +property "length/1 is always >= 0" do + check all list <- list_of(term()) do + assert length(list) >= 0 + end +end +``` + +With property-based testing, you specify a set of valid inputs (lists in the example above) for your code and verify that your code holds some property for values taken at random from the valid inputs. In the example above, the test takes many (usually around 100) values at random from the `list_of(term())` *generator* and verifies a property of `length/1`, that is, that `length/1` always returns a non-negative integer. A generator is just a `StreamData` generator, as we discussed in the previous section. + +### Shrinking + +Since we're generating lots of random inputs to test, inputs that cause failures are often complex and convoluted. Take this trivial example of a property: + +```elixir +property "list does not contain multiples of 4" do + check all list <- list_of(positive_integer()) do + refute Enum.any?(list, &(rem(&1, 4) == 0)) + end +end +``` + +When running this property, the failure might trigger for a list like this: + +```elixir +[19, 12, 6, 11, 2, 20, 10] +``` + +From this list, it's not easy to see why the test is failing (well, we know why because we wrote a doctored test that's supposed to fail). When running the property though, the failure that will be reported will look like this: + +``` +1) property list does not contain multiples of 7 (MyPropertyTest) + my_property_test.exs:6 + Failed with generated values (after 4 attempt(s)): + + list <- list_of(positive_integer()) + #=> [4] + + Expected false or nil, got true + code: refute Enum.any?(list, &(rem(&1, 4) == 0)) +``` + +This error shows the minimal generated value that triggers the failure, that is, `[4]`. The process of finding the minimal generated value that triggers a failure is called *shrinking*. All generators that come with `StreamData` generate values that "bundle" a way to shrink them so that property-based testing can use this to provide the shrinking functionality. Shrinking is a fundamental part of property-based testing as it takes out the *noise* of random-generated data to reduce the failing data to focused and easier-to-understand terms. + +### Using property-based testing in stream_data + +The core of property-based testing in stream_data is the `check all` macro. In this macro, you list a bunch of generators and filters (very similarly to how you would in `for` comprehensions) and then pass a body where you can verify that a property holds for the generated data. + +To make the `check all` macro available in your test, alongside importing all functions from `StreamData`, you can `use ExUnitProperties`: + +```elixir +defmodule MyPropertyTest do + use ExUnit.Case, async: true + use ExUnitProperties + + test "the in/2 operator works with lists" do + check all list <- list_of(term()), + list != [], + elem <- member_of(list) do + assert elem in list + end + end +end +``` + +As you can see, we can filter generated data (`list != []`) directly in the `check all` macro. We can also do simple assignments. The example above uses the `check all` macro inside a regular `test`. If you want that your properties are reported as "property" at the end of an ExUnit test run, you can use the `property` macro instead: + +```elixir +defmodule MyPropertyTest do + use ExUnit.Case, async: true + use ExUnitProperties + + property "the in/2 operator works with lists" do + check all list <- list_of(term()), + list != [], + elem <- member_of(list) do + assert elem in list + end + end +end +``` + +By doing this your properties will also be tagged with the `:property` tag, which means you will be able to do things like: + +```bash +mix test --only property +``` + +to run only properties. + +There's not much more to the mechanics of stream_data. Most of the work you will have to do revolves around finding good properties to test for your code and writing good generators for the data over which you want to test. Head over to [stream_data][]'s documentation for detailed documentation. + +### Advantages of property-based testing + +Using property-based testing has some advantages. First of all, it lets you test properties of your code over many more values than you otherwise would with example-based testing. While it's true that random data generation can't cover all the possible values that a piece of code can deal with, the confidence in your codebase can still increase over time because the property-based tests will likely generate different values on each run. Example-based testing means your test data will not change over time. + +Property-based testing however can also have a more powerful impact on the way you design software. When you start writing property-based tests, you will start thinking about what guarantees your code provides and what properties it satisfies. If you write properties before writing code, this can easily influence the way you write that code. + +### Learning resources + +Property-based testing is not something specific to Elixir. While having its roots in Haskell (check out the [original QuickCheck paper][quickcheck-paper] if you're interested), nowadays many languages have stable and usable implementations of it: Clojure has [test.check][], Python has [Hypothesis][], and many more. One of the most famous and complete tools for property-based testing exists for Erlang itself: [QuickCheck][] by Quviq is a complete commercial solution for property-based testing in Erlang of both stateless as well as stateful systems, and Quviq even provides a custom Erlang scheduler to test race conditions in your concurrent programs. + +A young but awesome book about property-based testing written by Fred Hebert is also available at [propertesting.com][]. This book is a *proper* (pun intended) guide to property-based testing and uses an Erlang library called [PropEr][]. However, the concepts and techniques perfectly apply to Elixir and stream_data as well. + +## Why include property-based testing in Elixir (and rewriting from scratch) + +The community has expressed some concern regarding two main things: why do we want to include a property-based testing tool in Elixir's standard library? And why write such a tool from scratch instead of using one of the existing Erlang or Elixir solutions? + +The answer to the first question is that we believe providing such a tool in the standard library will encourage developers to use property-based testing and ultimately improve their software and the way they write it. At the same time, we want to be able to use property-based testing to test the Elixir codebase itself (which already turned out great [in the past][quickcheck-pr]). + +The reasons for writing a new property-based testing library from scratch are best explained by José in [this ElixirForum post][elixirforum-post]: + +> * Since we want to bundle it as part of Elixir, the code should be open source with an appropriate license. +> * We wanted to add both data generation and property testing to Elixir. That's why the library is called stream_data instead of something named after property tests. The goal is to reduce the learning curve behind property testing by exposing the data generation aspect as streams, which is a known construct to most Elixir developers. We had this approach in mind for a while and the first library we saw leveraging this in practice was [@pragdave's pollution][pollution]. +> * Finally, since the core team are taking the responsibility of maintaining property testing as part of Elixir for potentially the rest of our lives, we want to have full understanding of every single line of code. This is non-negotiable as it guarantees we can continue to consistently improve the code as we move forward. +> +> We understand rolling our own implementation has its downsides, especially since it lacks maturity compared to alternatives, but we balance it by actively seeking input from knowledgeable folks and by listening to the feedback that comes from the community, which we are very thankful for. + +## Roadmap + +`stream_data` and the functionalities it includes are scheduled to be included in one of the next two Elixir releases, likely 1.6 but possibly 1.7. We have used the names `StreamData` and `ExUnitProperties` to avoid conflicts when those modules are eventually merged into Elixir. When merged, they will be renamed to the proper `Stream.Data` and `ExUnit.Properties` modules. Right now, all development is happening in the [stream_data][] repository, where we are discussing features and giving users a chance to try out the functionalities early on. We'd love for anyone to get involved in trying stream_data and we'd love feedback! + +**Update Jun/2020:** after careful consideration, the Elixir team decided to not include `StreamData` in Elixir itself, and keep it as package, as it is able to provide all of the necessary features without a need for direct integration with the language. + +[stream_data]: https://github.com/whatyouhide/stream_data +[quickcheck-paper]: http://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf +[test.check]: https://github.com/clojure/test.check +[Hypothesis]: https://github.com/HypothesisWorks/hypothesis-python +[QuickCheck]: http://quviq.com +[propertesting.com]: http://propertesting.com +[PropEr]: https://github.com/manopapad/proper +[quickcheck-pr]: https://github.com/elixir-lang/elixir/pull/5022#issuecomment-233195478 +[pollution]: https://github.com/pragdave/pollution +[elixirforum-post]: https://elixirforum.com/t/questions-about-property-testing-stream-data/9445/47 diff --git a/_posts/2018-01-17-elixir-v1-6-0-released.markdown b/_posts/2018-01-17-elixir-v1-6-0-released.markdown new file mode 100644 index 000000000..99bc8c441 --- /dev/null +++ b/_posts/2018-01-17-elixir-v1-6-0-released.markdown @@ -0,0 +1,157 @@ +--- +layout: post +title: Elixir v1.6 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.6 includes a code formatter, defguard, dynamic supervision and new module attributes that improves code quality and the developer experience +--- + +Elixir v1.6 includes new features, enhancements, and bug fixes. The main feature in this release is a code formatter. Important improvements can also be found in the standard library and in the Mix build tool. + +## Code formatter + +The big feature in Elixir v1.6 is the addition of [a code formatter](https://hexdocs.pm/elixir/Code.html#format_string!/2) and an accompanying `mix format` task that adds automatic formatting to your projects. + +The goal of the formatter is to automate the styling of codebases into a unique and consistent layout used across teams and the whole community. Code is now easier to write, as you no longer need to concern yourself with formatting rules. Code is also easier to read, as you no longer need to convert the styles of other developers in your mind. + +The formatter also helps new developers to learn the language by giving immediate feedback on code structure, and eases code reviews by allowing teams to focus on business rules and code quality rather than code style. + +To automatically format your codebase, you can run the [new `mix format` task](https://hexdocs.pm/mix/Mix.Tasks.Format.html). A `.formatter.exs` file may be added to your project root for rudimentary formatter configuration. The mix task also supports flags for CI integration. For instance, you can make your build or a Pull Request fail if the code is not formatted accordingly by using the `--check-formatted` flag. We also recommend developers to check their favorite editor and see if it already provides key bindings for `mix format`, allowing a file or a code snippet to be formatted without hassle. + +The Elixir codebase itself [has already been fully formatted](https://github.com/elixir-lang/elixir/issues/6643) and all further contributions are expected to contain formatted code. We recommend existing codebases to be formatted in steps. While the formatter will correctly handle long lines and complex expressions, refactoring the code by breaking those into variables or smaller functions as you format them will lead to overall cleaner and more readable codebases. + +## Dynamic Supervisor + +Supervisors in Elixir are responsible for starting, shutting down, and restarting child processes when things go wrong. Most of the interaction with supervisors happens through [the `Supervisor` module](https://hexdocs.pm/elixir/Supervisor.html) and it provides three main strategies: `:one_for_one`, `:rest_for_one` and `:one_for_all`. + +However, sometimes the children of a supervisor are not known upfront and are rather started dynamically. For example, if you are building a web server, you have each request being handled by a separate supervised process. Those cases were handled in the Supervisor module under a special strategy called `:simple_one_for_one`. + +Unfortunately, this special strategy changed the semantics of the supervisor in regards to initialization and shutdown. Plus some APIs expected different inputs or would be completely unavailable depending on the supervision strategy. + +Elixir v1.6 addresses this issue by introducing [a new `DynamicSupervisor` module](https://hexdocs.pm/elixir/DynamicSupervisor.html), which encapsulates the old `:simple_one_for_one` strategy and APIs in a proper module while allowing the documentation and API of the `Supervisor` module to focus on its main use cases. Having a separate `DynamicSupervisor` module also makes it simpler to add new features to the dynamic supervisor, such as the new `:max_children` option that limits the maximum number of children supervised dynamically. + +## `@deprecated` and `@since` attributes + +This release also introduces two new attributes associated with function definitions: `@deprecated` and `@since`. The former marks if a function or macro is deprecated, the latter annotates the version the API was introduced: + +```elixir +@doc "Breaks a collection into chunks" +@since "1.0.0" +@deprecated "Use chunk_every/2 instead" +def chunk(collection, chunk_size) do + chunk_every(collection, chunk_size) +end +``` + +The `mix xref` task was also updated to warn if your project calls deprecated code. So if a definition is marked as `@deprecated` and a module invokes it, a warning will be emitted during compilation. This effectively provides libraries and frameworks a mechanism to deprecate code without causing multiple warnings to be printed in runtime and without impacting performance. + +Note those attributes are not yet available to tools that generate documentation. Such functionality will be added in Elixir v1.7 once [Elixir adopts EEP-48](https://github.com/elixir-lang/elixir/issues/7198). We still recommend developers to start annotating their APIs so the information is already available when the tooling is updated. + +## `defguard` and `defguardp` + +Elixir provides the concepts of guards: expressions used alongside pattern matching to select a matching clause. Let's see an example straight from [Elixir's home page](https://elixir-lang.org): + +```elixir +def drive(%User{age: age}) when age >= 16 do + # Code that drives a car +end +``` + +`%User{age: age}` is matching on a `User` struct with an age field and `when age >= 16` is the guard. + +Since only a handful of constructs are [allowed in guards](https://hexdocs.pm/elixir/guards.html#content), if you were in a situation where you had to check the age to be more than or equal to 16 in multiple places, extracting the guard to a separate function would be [less than obvious and error prone](https://github.com/elixir-lang/elixir/issues/2469). To address those issues, [this release introduces `defguard/1` and `defguardp/1`](https://hexdocs.pm/elixir/Kernel.html#defguard/1): + +```elixir +defguard is_old_to_drive(age) when age >= 16 + +def drive(%User{age: age}) when is_old_to_drive(age) do + # Code that drives a car +end +``` + +## IEx improvements + +IEx also got its share of improvements. The new code formatter allows us to pretty print code snippets, types and specifications, improving the overall experience when exploring code through the terminal. + +The autocomplete mechanism also got smarter, being able to provide context autocompletion. For example, typing `t Enum.` and hitting TAB will autocomplete only the types in Enum (in contrast to all functions). Typing `b GenServer.` and hitting TAB will autocomplete only the behaviour callbacks. + +Finally, the breakpoint functionality added [in Elixir v1.5](https://elixir-lang.org/blog/2017/07/25/elixir-v1-5-0-released/) has been improved to support pattern matching and guards. For example, to pattern match on a function call when the first argument is the atom `:foo`, you may do: + +``` +iex> break! SomeFunction.call(:foo, _, _) +``` + +For more information, see [`IEx.break!/4`](https://hexdocs.pm/iex/IEx.html#break!/4). + +## mix xref + +[`mix xref`](https://hexdocs.pm/mix/Mix.Tasks.Xref.html) is a task added in Elixir v1.3 which provides general information about how modules and files in an application depend on each other. This release brings many improvements to `xref`, extending the reach of the analysis and helping developers digest the vast amount of data it produces. + +One of such additions is the `--include-siblings` option that can be given to all `xref` commands inside umbrella projects. For example, to find all of the callers of a given module or function of an application in an umbrella: + +``` +$ mix xref callers SomeModule --include-siblings +``` + +The `graph` command in `mix xref` now can also output general statistics about the graph. In [the hexpm project](https://github.com/hexpm/hexpm), you would get: + +``` +$ mix xref graph --format stats +Tracked files: 129 (nodes) +Compile dependencies: 256 (edges) +Structs dependencies: 46 (edges) +Runtime dependencies: 266 (edges) + +Top 10 files with most outgoing dependencies: + * test/support/factory.ex (18) + * lib/hexpm/accounts/user.ex (13) + * lib/hexpm/accounts/audit_log.ex (12) + * lib/hexpm/web/controllers/dashboard_controller.ex (12) + * lib/hexpm/repository/package.ex (12) + * lib/hexpm/repository/releases.ex (11) + * lib/hexpm/repository/release.ex (10) + * lib/hexpm/web/controllers/package_controller.ex (10) + * lib/mix/tasks/hexpm.stats.ex (9) + * lib/hexpm/repository/registry_builder.ex (9) + +Top 10 files with most incoming dependencies: + * lib/hexpm/web/web.ex (84) + * lib/hexpm/web/router.ex (29) + * lib/hexpm/web/controllers/controller_helpers.ex (29) + * lib/hexpm/web/controllers/auth_helpers.ex (28) + * lib/hexpm/web/views/view_helpers.ex (27) + * lib/hexpm/web/views/icons.ex (27) + * lib/hexpm/web/endpoint.ex (23) + * lib/hexpm/ecto/changeset.ex (22) + * lib/hexpm/accounts/user.ex (19) + * lib/hexpm/repo.ex (19) +``` + +`mix xref graph` also got the `--only-nodes` and `--label` options. The former asks Mix to only output file names (nodes) without the edges. The latter allows you to focus on certain relationships: + +``` +# To get all files that depend on lib/foo.ex +mix xref graph --sink lib/foo.ex --only-nodes + +# To get all files that depend on lib/foo.ex at compile time +mix xref graph --label compile --sink lib/foo.ex --only-nodes + +# To get all files lib/foo.ex depends on +mix xref graph --source lib/foo.ex --only-nodes + +# To limit statistics only to compile time dependencies +mix xref graph --format stats --label compile +``` + +Those improvements will help developers better understand the relationship between files and reveal potentially complex parts of their systems. + +Other improvements in Mix include [better compiler diagnostics](https://hexdocs.pm/mix/Mix.Task.Compiler.html) for editor integration, support for [the `--slowest N` flag in `mix test`](https://hexdocs.pm/mix/Mix.Tasks.Test.html) that shows the slowest tests in your suite, and a new [`mix profile.eprof` task](https://hexdocs.pm/mix/Mix.Tasks.Eprof.html) that provides time based profiling, complementing the existing [`mix profile.cprof` (count based)](https://hexdocs.pm/mix/Mix.Tasks.Profile.Cprof.html) and [`mix profile.fprof` (flame based)](https://hexdocs.pm/mix/Mix.Tasks.Profile.Fprof.html). + +## Summing up + +The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.6.0). There are many other exciting changes, such as compiler enhancements to better track dependencies, leading to less files being recompiled whenever there are changes in project, and overall faster compilation. + +Work on Elixir v1.7 has already started. We still welcome developers to try out the [previously announced StreamData library](https://elixir-lang.org/blog/2017/10/31/stream-data-property-based-testing-and-data-generation-for-elixir/), that aims to bring data generation and property-based testing to Elixir. The other [features scheduled for v1.7 can be found in the issues tracker](https://github.com/elixir-lang/elixir/issues). + +Don't forget to check [the Install section](/install.html) to get Elixir installed and our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. diff --git a/_posts/2018-06-28-gsoc-projects.markdown b/_posts/2018-06-28-gsoc-projects.markdown new file mode 100644 index 000000000..168f6126e --- /dev/null +++ b/_posts/2018-06-28-gsoc-projects.markdown @@ -0,0 +1,86 @@ +--- +layout: post +title: Google Summer of Code 2018 projects +authors: +- Andrea Leopardi +category: Announcements +excerpt: Once again Elixir is participating in Google Summer of Code 2018. In this post, we'll have a look at the active projects. +--- + +Like previous years, the Elixir community is happy to participate in [Google Summer of Code][gsoc] 2018. We are currently working on four different projects. Let's have a look at them. + +## StreamData integration with typespecs + +*Student: Nikola Jichev* + +[StreamData][stream_data] is a data-generation and property-based testing library for Elixir. The goal of this GSoC project is to integrate StreamData with typespecs. + +The data-generation side of StreamData provides tools to generate random data through composable generators. For example, you could generate keyword lists like this: + +```elixir +import StreamData + +keywords_generator = list_of({atom(:alphanumeric), term()}) + +Enum.take(keywords_generator, 2) +#=> [[_: [true]], [tm: 2, h: %{}]] +``` + +In many cases, it would be useful to be able to generate such random data starting from already existing or user-defined types. For example, Elixir already provides a built-in `keyword/0` type for keyword lists defined roughly as: + +```elixir +@type keyword() :: [{atom(), any()}] +``` + +The goal of the first part of this GSoC project is to provide StreamData with the ability to create data generators from type definitions. The API is not yet defined, but in this case, it could look something like the following: + +```elixir +import StreamData + +keywords_generator = from_type(keyword/0) + +Enum.take(keywords_generator, 2) +#=> [[_: [true]], [tm: 2, h: %{}]] +``` + +In the second part of the GSoC project, the aim is to be able to property-test functions with specs automatically. + +```elixir +@spec has_key?(keyword(), atom()) :: boolean() +def has_key?(keyword, key) do + # ... +end +``` + +The first part of the project focuses on generating data from types, so we know how to generate function arguments. The missing piece is **validating** that a given term *belongs to* a given type. For example, in the snippet above, we want to be able to check if a term is a `boolean()`. Once we're able to do this, automatic spec validation will be straightforward: it will be a matter of generating random arguments for the given function, calling the function with those arguments, and asserting that the returned value belongs to the return type defined in the spec. + +This kind of property-based testing doesn't test for *correctness*. In the snippet above, `has_key?/2` could be implemented to ignore arguments always return `false` and the automatic spec validation would pass since `false` is always a boolean. However, this is a kind of **smoke testing** useful for discovering inconsistencies in the arguments and return values of functions. + +## Tensorflex: Tensorflow bindings for the Elixir programming language + +*Student: Anshuman Chhabra* + +Currently, there is a lack of machine learning tools and frameworks for Elixir. With the number of programmers learning/using machine learning only set to grow, supporting machine learning capabilities is essential for any programming language. Moreover, there are discussions on [ElixirForum][elixirforum] regarding this and recent talks given at ElixirConf that reflect the need for Elixir to provide machine learning capabilities. + +This project's goal is Tensorflex, an Elixir machine learning framework similar to [Keras for Python][keras]. Keras uses Tensorflow as a backend for doing all the machine learning. Tensorflex will use Using Native Implemented Functions (NIFs) and the Tensorflow C API as a backend to provide a low-level API. This low-level API will then be used to write a Keras-like framework in the form of a high-level API. This will allow Elixir developers to write expedient and efficient machine learning code in Elixir. + +## Dialyzer task for Elixir + +*Student: Gabriel Gatu* + +Dialyzer is a discrepancy analyzer that ships as part of Erlang/OTP. Currently, there are two projects that add Dialyzer support to Elixir applications: [dialyxir][] and [dialyzex][]. The goal of this project is to bring ideas from both projects into Elixir itself in order to make using Dialyzer in Elixir projects easier. The task we aim to add to Elixir will focus on two main features: better user experience (in particular, better error messages and formatting) and the ability to analyze projects incrementally. + +## ElixirBench + +*Student: Tallys Martins* + +ElixirBench aims to be a service to monitor performance of Elixir projects. The goal of the GSoC project is to bring ElixirBench up and have it run nightly performance monitoring of significant Elixir projects (including Elixir itself). The end goal is to have a platform that, given a project from GitHub, will monitor the performance of new releases of that project and look for performance regressions. The benchmarking process will be controlled through a configuration file that will specify the benchmark scripts to run. + +We have high hopes for this tool as we see value in it for the whole community and for core Elixir projects alike. + +[gsoc]: https://summerofcode.withgoogle.com +[stream_data]: https://github.com/whatyouhide/stream_data +[elixirforum]: https://elixirforum.com +[keras]: https://keras.io +[dialyxir]: https://github.com/jeremyjh/dialyxir +[dialyzex]: https://github.com/Comcast/dialyzex diff --git a/_posts/2018-07-25-elixir-v1-7-0-released.markdown b/_posts/2018-07-25-elixir-v1-7-0-released.markdown new file mode 100644 index 000000000..51e8f7a66 --- /dev/null +++ b/_posts/2018-07-25-elixir-v1-7-0-released.markdown @@ -0,0 +1,157 @@ +--- +layout: post +title: Elixir v1.7 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.7 includes many quality of life improvements, focusing on documentation, Logger and ExUnit, as well as a new Elixir Core team member! +--- + +A new semester has started, which means it is time for a new Elixir release! This release brings quality of life improvements to the documentation, to error handling, to logger reporting, and to ExUnit, Elixir's testing library. + +We are also glad to welcome Michał Muskała to the Elixir Core team. Prior to joining the team, he was [a member of the Ecto team](https://github.com/elixir-ecto/ecto), he has made [plenty of contributions to Elixir](https://github.com/elixir-lang/elixir/pulls?utf8=%E2%9C%93&q=is%3Apr+author%3Amichalmuskala), often to improve performance, and [is a frequent to contribute to Erlang/OTP too](https://github.com/erlang/otp/pulls?utf8=%E2%9C%93&q=is%3Apr+author%3Amichalmuskala)! + +## Documentation metadata + +Elixir v1.7 implements [EEP 48](http://www.erlang.org/eep/eeps/eep-0048.html). EEP 48 aims to bring documentation interoperability across all languages running on the Erlang VM. + +Furthermore, EEP 48 introduces the ability to annotate documentation with metadata, which we have made possible to leverage from Elixir: + +```elixir +@moduledoc "A brand new module" +@moduledoc authors: ["Jane", "Mary"], since: "1.4.0" +``` + +Metadata can be given to `@moduledoc`, `@typedoc` and `@doc`. + +We have updated [the ExDoc tool](https://github.com/elixir-lang/ex_doc) to start leveraging metadata in order to provide better documentation for developers. Some of the improvements include: + + * Deprecated modules, functions, callbacks and types have a warning automatically attached to them. [See the deprecated `Behaviour` module as an example](https://hexdocs.pm/elixir/Behaviour.html) + + * Functions, macros, callbacks and types now include the version in which they were added. For example, [see the top right corner of the `defguard` docs](https://hexdocs.pm/elixir/Kernel.html#defguard/1) + + * Future Elixir versions will [include its own section for guards in the documentation and in the sidebar](https://hexdocs.pm/elixir/main/Kernel.html#guards). We are currently exploring ways to [generalize this feature in ExDoc itself](https://github.com/elixir-lang/ex_doc/issues/876) + +Elixir's interactive shell, IEx, has also been updated to print metadata: + +![IEx metadata](/images/contents/iex-metadata.png) + +While Elixir allows any metadata to be given, those tools currently exhibit only `:deprecated` and `:since`. Other keys may be shown in the future. + +Those improvements are not exclusive to the standard library, they are available to every Elixir library and application. We hope one day they will be available to all applications running on the Erlang VM too. + +To access the new documentation format, developers should use [`Code.fetch_docs/1`](https://hexdocs.pm/elixir/Code.html#fetch_docs/1). We have always been proud of treating documentation as a first-class citizen and the ability to add structured information to the documentation is a further step in this direction. + +## The `__STACKTRACE__` construct + +Erlang/OTP 21.0 introduces a new way to retrieve the stacktrace that is lexically scoped and no longer relies on side-effects like `System.stacktrace/0` does. Before one would write: + +```elixir +try do + ... something that may fail ... +rescue + exception -> + log(exception, System.stacktrace()) + reraise(exception, System.stacktrace()) +end +``` + +In Elixir v1.7, this can be written as: + +```elixir +try do + ... something that may fail ... +rescue + exception -> + log(exception, __STACKTRACE__) + reraise(exception, __STACKTRACE__) +end +``` + +This change may also yield performance improvements in the future, since the lexical scope allows us to track precisely when a stacktrace is used and we no longer need to keep references to stacktrace entries after the `try` construct finishes. + +Other parts of the exception system have also been improved. For example, more information is provided in certain occurrences of `ArgumentError`, `ArithmeticError` and `KeyError` messages. + +## Erlang/OTP logger integration + +Erlang/OTP 21 includes a new `:logger` module. Elixir v1.7 fully integrates with the new `:logger` and leverages its metadata system. The `Logger.Translator` mechanism has also been improved to export metadata, allowing custom Logger backends to leverage information such as: + + * `:crash_reason` - a two-element tuple with the throw/error/exit reason as the first argument and the stacktrace as the second + + * `:initial_call` - the initial call that started the process + + * `:registered_name` - the process' registered name as an atom + +We recommend Elixir libraries that previously hooked into Erlang's `:error_logger` to hook into `Logger` instead, in order to support all current and future Erlang/OTP versions. + +## Logger compile-time purging + +Previously, Logger macros such as `debug`, `info`, and so on would always evaluate their arguments, even when nothing would be logged. From Elixir v1.7 the arguments are only evaluated when the message is logged. + +The Logger configuration system also accepts a new option called `:compile_time_purge_matching` that allows you to remove log calls with specific compile-time metadata. For example, to remove all logger calls from application `:foo` with level lower than `:info`, as well as remove all logger calls from `Bar.foo/3`, you can use the following configuration: + +```elixir +config :logger, + compile_time_purge_matching: [ + [application: :foo, level_lower_than: :info], + [module: Bar, function: "foo/3"] + ] +``` + +## ExUnit improvements + +[ExUnit](https://hexdocs.pm/ex_unit/) is Elixir's unit testing library. ExUnit has always leveraged Elixir macros to provide excellent error reports when a failure happens. For example, the following code: + +```elixir +assert "fox jumps over the lazy dog" == "brown fox jumps over the dog" +``` + +will fail with the following report: + +![ExUnit Diff](/images/contents/exunit-diff.png) + +The `assert` macro is able to look at the code, extract the current file, the line, extract the operands and show a diff between the data structures alongside the stacktrace when the assertion fails. + +However, sometimes we need to write assertions such as `assert some_function(expr1, var2)`. When such assertion fails, we usually have to re-run the tests, now debugging or printing the values of `expr1` and `var2`. In Elixir v1.7, when a "bare" assertion fails, we will print the value of each argument individually. For a simple example such as `assert some_vars(1 + 2, 3 + 4)`, we get this report: + +![ExUnit Bare Assertion Diff](/images/contents/exunit-bare-assertion-diff.png) + +We have also [added coloring and diffing to doctests](https://hexdocs.pm/ex_unit/ExUnit.DocTest.html#content). + +While ExUnit is our test framework, Mix is our build tool. Developers typically run their tests by calling `mix test`. + +On the `mix test` side of things, there is a new `--failed` flag that runs all tests that failed the last time they ran. Finally, coverage reports generated with `mix test --cover` include a summary out of the box: + +``` +Generating cover results ... + +Percentage | Module +-----------|-------------------------- + 100.00% | Plug.Exception.Any + 100.00% | Plug.Adapters.Cowboy2.Stream + 100.00% | Collectable.Plug.Conn + 100.00% | Plug.Crypto.KeyGenerator + 100.00% | Plug.Parsers + 100.00% | Plug.Head + 100.00% | Plug.Router.Utils + 100.00% | Plug.RequestId + ... | ... +-----------|-------------------------- + 77.19% | Total +``` + +## Summing up + +We are really proud of this release, as it focuses mostly on quality of life improvements, instead of flashy new features. As Elixir continues to mature, expect more releases like this one. The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.7.0). + +We have also seen important developments in other areas not directly related to the Elixir codebase: + + * We have added [a "Development" section](https://elixir-lang.org/development.html) to the website, that outlines the Elixir team structure and goals + + * [Elixir now has its own mini-documentary](https://www.youtube.com/watch?v=lxYFOM3UJzo) by Honeypot + + * We have already highlighted some of the improvements in the ExDoc tool. Another improvement worth mentioning is the syntax highlighting is now done in Elixir itself, via the [Makeup](https://github.com/tmbb/makeup) library. This gives us more control over the grammar, the style, and improves load times. If you would like to add support for other languages, [reach out](https://github.com/tmbb/makeup)! + +Finally, don't forget [ElixirConf US](https://elixirconf.com/) is coming soon, in Bellevue, WA, September 4-7. Last year my keynote focused on the last 5 years with Elixir. This year we are ready to look into the 5 years ahead. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. diff --git a/_posts/2019-01-14-elixir-v1-8-0-released.markdown b/_posts/2019-01-14-elixir-v1-8-0-released.markdown new file mode 100644 index 000000000..6da84a7dc --- /dev/null +++ b/_posts/2019-01-14-elixir-v1-8-0-released.markdown @@ -0,0 +1,83 @@ +--- +layout: post +title: Elixir v1.8 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.8 comes with many improvements at the infrastructure level, improving compilation time, speeding up common patterns, and adding features around introspection of the system. +--- + +Elixir v1.8 comes with many improvements at the infrastructure level, improving compilation time, speeding up common patterns, and adding features around introspection of the system. + +## Custom struct inspections + +Elixir now provides a derivable implementation of the `Inspect` protocol. In a nutshell, this means it is really easy to filter data from your data structures whenever they are inspected. For example, imagine you have a user struct with security and privacy sensitive information: + +```elixir +defmodule User do + defstruct [:id, :name, :age, :email, :encrypted_password] +end +``` + +By default, if you inspect a user via `inspect(user)`, it will include all fields. This can cause fields such as `:email` and `:encrypted_password` to appear in logs, error reports, etc. You could always define a custom implementation of the `Inspect` protocol for such cases but Elixir v1.8 makes it simpler by allowing you to derive the `Inspect` protocol: + +```elixir +defmodule User do + @derive {Inspect, only: [:id, :name, :age]} + defstruct [:id, :name, :age, :email, :encrypted_password] +end +``` + +Now all user structs will be printed with all remaining fields collapsed: + + #User + +You can also pass `@derive {Inspect, except: [...]}` in case you want to keep all fields by default and exclude only some. + +## Time zone database support + +In Elixir v1.3, Elixir added four types, known as Calendar types, to work with dates and times: `Time`, `Date`, `NaiveDateTime` (without time zone), and `DateTime` (with time zone). Over the last versions we have added many enhancements to the Calendar types but the `DateTime` module always evolved at a slower pace since Elixir did not provide an API for time zone databases. + +Elixir v1.8 now defines a `Calendar.TimeZoneDatabase` behaviour, allowing developers to bring in their own time zone databases. By defining an explicit contract for time zone behaviours, Elixir can now extend the `DateTime` API, adding functions such as `DateTime.shift_zone/3`. By default, Elixir ships with a time zone database called `Calendar.UTCOnlyTimeZoneDatabase` that only handles UTC. + +Other Calendar related improvements include the addition of `Date.day_of_year/1`, `Date.quarter_of_year/1`, `Date.year_of_era/1`, and `Date.day_of_era/1`. + +## Faster compilation and other performance improvements + +Due to improvements to the compiler made over the last year, Elixir v1.8 should compile code about 5% faster on average. This is yet another release where we have been able to reduce compilation times and provide a more joyful development experience to everyone. + +The compiler also emits more efficient code for range checks in guards (such as `x in y..z`), for charlists with interpolation (such as `'foo #{bar} baz'`), and when working with records via the `Record` module. + +Finally, EEx templates got their own share of optimizations, emitting more compact code that runs faster. + +## Improved instrumentation and ownership with `$callers` + +The `Task` module is one of the most common ways to spawn light-weight processes to perform work concurrently. Whenever you spawn a new process, Elixir annotates the parent of that process through the `$ancestors` key. This information can be used by instrumentation tools to track the relationship between events occurring within multiple processes. However, many times, tracking only the `$ancestors` is not enough. + +For example, we recommend developers to always start tasks under a supervisor. This provides more visibility and allows us to control how those tasks are terminated when a node shuts down. In your code, this can be done by invoking something like: `Task.Supervisor.start_child(MySupervisor, task_specification)`. This means that, although your code is the one who invokes the task, the actual parent of the task would be the supervisor, as the supervisor is the one spawning it. We would list the supervisor as one of the `$ancestors` for the task, but the relationship between your code and the task is lost. + +In Elixir v1.8, we now track the relationship between your code and the task via the `$callers` key in the process dictionary, which aligns well with the existing `$ancestors` key. Therefore, assuming the `Task.Supervisor` call above, we have: + + [your code] -- calls --> [supervisor] ---- spawns --> [task] + +which means we store the following relationships: + + [your code] [supervisor] <-- ancestor -- [task] + ^ | + |--------------------- caller ---------------------| + +When a task is spawned directly from your code, without a supervisor, then the process running your code will be listed under both `$ancestors` and `$callers`. + +This small feature is very powerful. It allows instrumentation and monitoring tools to better track and relate the events happening in your system. This feature can also be used by tools like the "Ecto Sandbox". The "Ecto Sandbox" allows developers to run tests concurrently against the database, by using transactions and an ownership mechanism where each process explicitly gets a connection assigned to it. Without `$callers`, every time you spawned a task that queries the database, the task would not know its caller, and therefore it would be unable to know which connection was assigned to it. This often meant features that rely on tasks could not be tested concurrently. With `$callers`, figuring out this relationship is trivial and you have more tests using the full power of your machine. + +## Summing up + +We are really proud of this release (as usual!) which brings many improvements at the infrastructure level. Those improvements were designed with feedback from the community and from the many different companies using Elixir in production. The full list of changes is available in our [release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.8.0). + +There is only one last major feature planned for upcoming Elixir versions, which is the addition of `mix release` to Elixir itself, streamlining the experience provided by packages like [distillery](https://github.com/bitwalker/distillery). With `mix release`, a developer can bundle the VM and all compiled code in a single directory, which can then be packaged and sent to production. We are glad to say the [work on this feature has already started](https://github.com/elixir-lang/elixir/issues/8612). + +During [my keynote at ElixirConf 2018 US](https://www.youtube.com/watch?v=suOzNeMJXl0), I talked about the next five years for Elixir and much of the emphasis is put on the community. Elixir was designed to be an extensible language and therefore the work on the language itself is meant to reduce with time, which we have seen in the last two releases. We trust the community to continue building on this solid foundation, bringing new challenges to the ecosystem and taking the language to new domains. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Your turn. :) \ No newline at end of file diff --git a/_posts/2019-02-25-mint-a-new-http-library-for-elixir.markdown b/_posts/2019-02-25-mint-a-new-http-library-for-elixir.markdown new file mode 100644 index 000000000..3868b94e6 --- /dev/null +++ b/_posts/2019-02-25-mint-a-new-http-library-for-elixir.markdown @@ -0,0 +1,80 @@ +--- +layout: post +title: Mint, a new HTTP client for Elixir +authors: +- Eric Meadows-Jönsson +category: Announcements +excerpt: Mint is a new low-level HTTP client that aims to provide a small and functional core that others can build on top. +--- + +[Mint](https://github.com/ericmj/mint) is a new low-level HTTP client that aims to provide a small and functional core that others can build on top. Mint is connection based: each connection is a single struct with an associated socket belonging to the process that started the connection. Since no extra processes are started for the connection, you can choose the process architecture that better fits your application. + +To validate this we built out the library with a common API supporting both HTTP/1 and HTTP/2 with automatic version negotiation. In addition, Mint comes with a [CA certificate store](https://github.com/ericmj/castore) to do safe by default HTTPS connections. + +## Connections without processes + +Mint HTTP connections are managed directly in the process that starts the connection, which means no connection pool is used nor new processes spawned when a connection is opened. This allows the user of the library to build their own process structure that fits their application. + +With Mint each connection has a single immutable data structure that the user needs to manage. Mint uses ["active mode"](http://www.erlang.org/doc/man/inet.html#setopts-2) sockets. This means data and events from the socket are sent as messages to the process that started the connection. The user passes the messages to the `stream/2` function that returns the updated connection and a list of "responses". Responses are streamed back which means you won't receive a single full HTTP response back from `stream/2`, instead the response is returned in partial response chunks. A chunk can be the status line, HTTP headers, or part of the response body. + +Let's look at an example of sending a request with Mint: + +```elixir +iex(1)> {:ok, conn} = Mint.HTTP.connect(:http, "httpbin.org", 80) +iex(2)> {:ok, conn, request_ref} = Mint.HTTP.request(conn, "GET", "/", [], "") +iex(3)> receive do +...(3)> message -> +...(3)> IO.inspect(message, label: :message) +...(3)> {:ok, conn, responses} = Mint.HTTP.stream(conn, message) +...(3)> IO.inspect(responses, label: :responses) +...(3)> end +message: {:tcp, #Port<0.8>, "HTTP/1.1 200 OK\r\n" <> ...} +responses: [ + {:status, #Reference<...>, 200}, + {:headers, #Reference<...>, [{"connection", "keep-alive"}, ...}, + {:data, #Reference<...>, "" <> ...}, + {:done, #Reference<...>} +] +``` + +As we can see all calls to `Mint.HTTP` functions return an updated `conn` which holds the state for the connection. It is important to carry on the `conn` to the next function call or the state will be corrupted. + +On line 2 we send a request to the server. A reference to the request is returned: this reference is useful when sending concurrent requests, either with HTTP/1 pipelining or with HTTP/2 multiplexed streams. + +Next we start a receive block waiting for a TCP active mode message and pass it to `stream/2`. The message is parsed and the response to the request is returned. As you can see the response is split over multiple tuples: `:status`, `:headers`, `:data`, and `:done`. This is because Mint was built from the ground with streaming in mind. The parts of the response will be returned continuously as TCP messages are passed to `stream/2` so that we don't have to wait for the full response to complete before starting to process it. + +If the response body is larger than a single packet `stream/2` may return multiple `:data` tuples and if the response includes trailing headers multiple `:headers` will be returned. When the response is complete `:done` will be returned. + +Note that if you send concurrent requests on a HTTP/2 connection responses can be returned interleaved from the requests using HTTP/2's stream multiplexing. Additionally, responses can be spread over multiple messages so we may need to continually receive messages and pass them to `stream/2`. + +See more examples on how to use Mint in the [documentation](https://hexdocs.pm/mint). + +## Why process-less? + +Mint may seem more cumbersome to use than most other HTTP libraries you have used and that is true in many ways. But by providing a low-level API without a predetermined process architecture it gives more flexibility to the user of the library. + +Many times you do not need a general purpose connection pool and can avoid the additional complexity, single point of failure, and potential performance bottlenecks that it brings. For example, if you are building quick CLI scripts, you most likely don't need a pool and performing a single one-off request with Mint is good enough. + +Another good use case for Mint is [GenStage](https://github.com/elixir-lang/gen_stage). If you write GenStage pipelines, it is most likely that you have a pool of producers that fetch data from external sources via HTTP. If you are using a high-level HTTP library, that comes with its own pool, now you have two pools, one of GenStage producers and another from the HTTP library. With Mint, you can have each GenStage producer manage its own connection, reducing overhead and simplifying the code. + +Of course, none of this stops you from building a connection pool on top of Mint. The point is exactly that Mint won't impose an architecture onto you. At the end of the day, we hope Mint will be a useful building block for more complex scenario and use cases. + +## HTTP/1 and HTTP/2 + +The `Mint.HTTP` module has a single interface for both HTTP/1 and HTTP/2 connections and performs version negotiation on HTTPS connections, HTTP connections default to HTTP/1. You can specify which HTTP version you want to use or use the `Mint.HTTP1` or `Mint.HTTP2` modules directly if you want to use version-specific features. + +## Safe-by-default HTTPS + +When connecting over HTTPS, Mint will perform certificate verification by default. We believe it's crucial that an HTTP library defaults to be secure out of the box. + +Mint uses an optional dependency on [CAStore](https://github.com/ericmj/castore) to provide certificates from [Mozilla's CA Certificate Store](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/). + +You can of course tweak specific SSL settings without re-building the safe defaults yourself. + +## Current state of the library + +The first version of Mint has just been released. It is an experimental library trying a new approach to building HTTP libraries so don't expect a fully stable API yet. + +Use Mint to explore new ideas for HTTP connection management and building higher level clients on top of Mint. In the future connection pooling and a higher level API may be added to supplement the current low level API, either directly to Mint or via different libraries. + +*Note:* Mint is being announced in the official Elixir blog because it was originally being considered for inclusion in Elixir itself. However, at some point the Elixir team decided it doesn't make sense to include an HTTP client in Elixir itself, at least as long as Erlang/OTP ships with a client too. Mint is not maintained by the Elixir team, although it is maintained by Eric and Andrea, who are part of the team. diff --git a/_posts/2019-06-24-elixir-v1-9-0-released.markdown b/_posts/2019-06-24-elixir-v1-9-0-released.markdown new file mode 100644 index 000000000..b1b72fd4c --- /dev/null +++ b/_posts/2019-06-24-elixir-v1-9-0-released.markdown @@ -0,0 +1,102 @@ +--- +layout: post +title: Elixir v1.9 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.9 is out with releases support, improved configuration and more. +--- + +Elixir v1.9 is out with releases support, improved configuration, and more. + +We are also glad to announce [Fernando Tapia Rico](https://github.com/fertapric) has joined the Elixir Core Team. Fernando has been extremely helpful in keeping the issues tracker tidy, by fixing bugs and improving Elixir in many different areas, such as the code formatter, IEx, the compiler, and others. + +Now let's take a look at what's new in this new version. + +## Releases + +The main feature in Elixir v1.9 is the addition of releases. A release is a self-contained directory that consists of your application code, all of its dependencies, plus the whole Erlang Virtual Machine (VM) and runtime. Once a release is assembled, it can be packaged and deployed to a target as long as the target runs on the same operating system (OS) distribution and version as the machine running the [`mix release`](https://hexdocs.pm/mix/Mix.Tasks.Release.html) command. + +Releases have always been part of the Elixir community thanks to Paul Schoenfelder's work on [Distillery](https://github.com/bitwalker/distillery) (and EXRM before that). Distillery was announced in July 2016. Then in 2017, [DockYard](https://dockyard.com/) hired Paul to work on improving deployments, an effort that would lead to [Distillery 2.0](https://dockyard.com/blog/2018/08/23/announcing-distillery-2-0). Distillery 2.0 provided important answers in areas where the community was struggling to establish conventions and best practices, such as configuration. + +At the beginning of this year, thanks to [Plataformatec](http://plataformatec.com.br/), I was able to prioritize the work on bringing releases directly into Elixir. Paul was aware that we wanted to have releases in Elixir itself and during [ElixirConf 2018](https://elixirconf.com) I announced that releases was the last planned feature for Elixir. + +The goal of Elixir releases was to double down on the most important concepts provided by Distillery and provide extensions points for the other bits the community may find important. [Paul](http://github.com/bitwalker/) and [Tristan](https://github.com/tsloughter) (who maintains [Erlang's relx](https://github.com/erlware/relx)) provided excellent feedback on Elixir's implementation, which we are very thankful for. [The Hex package manager is already using releases in production](https://dashbit.co/blog/updating-hex-pm-to-use-elixir-releases) and we also got feedback from other companies doing the same. + +Enough background, let's see why you would want to use releases and how to assemble one. + +### Why releases? + +Releases allow developers to precompile and package all of their code and the runtime into a single unit. The benefits of releases are: + + * Code preloading. The VM has two mechanisms for loading code: interactive and embedded. By default, it runs in the interactive mode which dynamically loads modules when they are used for the first time. The first time your application calls `Enum.map/2`, the VM will find the `Enum` module and load it. There's a downside. When you start a new server in production, it may need to load many other modules, causing the first requests to have an unusual spike in response time. Releases run in embedded mode, which loads all available modules upfront, guaranteeing your system is ready to handle requests after booting. + + * Configuration and customization. Releases give developers fine grained control over system configuration and the VM flags used to start the system. + + * Self-contained. A release does not require the source code to be included in your production artifacts. All of the code is precompiled and packaged. Releases do not even require Erlang or Elixir in your servers, as they include the Erlang VM and its runtime by default. Furthermore, both Erlang and Elixir standard libraries are stripped to bring only the parts you are actually using. + + * Multiple releases. You can assemble different releases with different configuration per application or even with different applications altogether. + + * Management scripts. Releases come with scripts to start, restart, connect to the running system remotely, execute RPC calls, run as daemon, run as a Windows service, and more. + +### 1, 2, 3: released assembled! + +You can start a new project and assemble a release for it in three easy steps: + + $ mix new my_app + $ cd my_app + $ MIX_ENV=prod mix release + +A release will be assembled in `_build/prod/rel/my_app`. Inside the release, there will be a `bin/my_app` file which is the entry point to your system. It supports multiple commands, such as: + + * `bin/my_app start`, `bin/my_app start_iex`, `bin/my_app restart`, and `bin/my_app stop` - for general management of the release + + * `bin/my_app rpc COMMAND` and `bin/my_app remote` - for running commands on the running system or to connect to the running system + + * `bin/my_app eval COMMAND` - to start a fresh system that runs a single command and then shuts down + + * `bin/my_app daemon` and `bin/my_app daemon_iex` - to start the system as a daemon on Unix-like systems + + * `bin/my_app install` - to install the system as a service on Windows machines + +### Hooks and Configuration + +Releases also provide built-in hooks for configuring almost every need of the production system: + + * `config/config.exs` (and `config/prod.exs`) - provides build-time application configuration, which is executed when the release is assembled + + * `config/releases.exs` - provides runtime application configuration. It is executed every time the release boots and is further extensible via config providers + + * `rel/vm.args.eex` - a template file that is copied into every release and provides static configuration of the Erlang Virtual Machine and other runtime flags + + * `rel/env.sh.eex` and `rel/env.bat.eex` - template files that are copied into every release and executed on every command to set up environment variables, including ones specific to the VM, and the general environment + +We have written [extensive documentation on releases](https://hexdocs.pm/mix/Mix.Tasks.Release.html), so we recommend checking it out for more information. + +## Configuration + +We also use the work on releases to streamline Elixir's configuration API. A new `Config` module has been added to Elixir. The previous configuration API, `Mix.Config`, was part of the Mix build tool. However, since releases provide runtime configuration and Mix is not included in releases, we ported the `Mix.Config` API to Elixir. In other words, `use Mix.Config` has been soft-deprecated in favor of `import Config`. + +Another important change related to configuration is that `mix new` will no longer generate a `config/config.exs` file. [Relying on configuration is undesired for most libraries](https://hexdocs.pm/elixir/library-guidelines.html#avoid-application-configuration) and the generated config files pushed library authors in the wrong direction. Furthermore, `mix new --umbrella` will no longer generate a configuration for each child app, instead all configuration should be declared in the umbrella root. That's how it has always behaved, we are now making it explicit. + +## Other improvements + +There are many other enhancements in Elixir v1.9. The Elixir CLI got a handful of new options in order to best support releases. `Logger` now computes its sync/async/discard thresholds in a decentralized fashion, reducing contention. `EEx` (Embedded Elixir) templates support more complex expressions than before. Finally, there is a new `~U` sigil for working with UTC DateTimes as well as new functions in the `File`, `Registry`, and `System` modules. + +## What's next? + +As mentioned earlier, releases was the last planned feature for Elixir. We don't have any major user-facing feature in the works nor planned. I know for certain some will consider this fact the most excing part of this announcement! + +Of course, it does not mean that v1.9 is the last Elixir version. We will continue shipping new releases every 6 months with enhancements, bug fixes and improvements. You can see the [Issues Tracker](http://github.com/elixir-lang/elixir/issues) for more details. + +We also are working on some structural changes. One of them is move the `mix xref` pass straight into the compiler, which would allow us to emit undefined function and deprecation warnings in more places. We are also considering a move to [Cirrus-CI](https://cirrus-ci.org/), so we can test Elixir on Windows, Unix, and FreeBSD through a single service. + +It is also important to highlight that there are two main reasons why we can afford to have an empty backlog. + +First of all, Elixir is built on top of Erlang/OTP and we simply leverage all of the work done by Ericsson and the OTP team on the runtime and Virtual Machine. The Elixir team has always aimed to contribute back as much as possible and those contributions have increased in the last years. + +Second, Elixir was designed to be an extensible language. The same tools and abstractions we used to create and enhance the language are also available to libraries and frameworks. This means the community can continue to improve the ecosystem without a need to change the language itself, which would effectively become a bottleneck for progress. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. We have also updated our [advanced Mix & OTP](https://hexdocs.pm/elixir/introduction-to-mix.html) to talk about releases. If you are looking for a more fast paced introduction to the language, see the [How I Start: Elixir](http://howistart.org/posts/elixir/1/index.html) tutorial, which has also been brought to the latest and greatest. + +Have fun! diff --git a/_posts/2020-01-27-elixir-v1-10-0-released.markdown b/_posts/2020-01-27-elixir-v1-10-0-released.markdown new file mode 100644 index 000000000..f2eac58ce --- /dev/null +++ b/_posts/2020-01-27-elixir-v1-10-0-released.markdown @@ -0,0 +1,171 @@ +--- +layout: post +title: Elixir v1.10 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.10 is out with standard library, compiler, and releases improvements. +--- + +Elixir v1.10 comes with improvements to the standard library, the compiler, as well as several additions to the [`mix release` feature added in v1.9](/blog/2019/06/24/elixir-v1-9-0-released/). In particular, this version adds a lot of polish to existing features, such as our configuration system and our sorting APIs. + +Also note Elixir v1.10 requires Erlang/OTP 21+. This allows us to provide tighter integration with Erlang/OTP's new logger. This means that the logger level, logger metadata, as well as all log messages are now shared between Erlang and Elixir APIs. + +Let's take a look at what else is new. + +## Releases improvements + +Elixir v1.9 introduced releases as a mechanism to package self-contained applications. Elixir v1.10 further improves releases with bug fixes and new enhancements based on feedback we got from the community. The highlights are: + + * Allow the dual boot system of releases to be disabled on environments that are boot-time sensitive, such as embedded devices + + * Track and raise if compile-time configuration is set or changes at runtime (more in the next section) + + * Support overlays to easily add extra files to a packaged releases + + * Allow `RELEASE_DISTRIBUTION` to be set to `none` in order to fully disable distribution + + * Add a built-in `:tar` step that automatically packages releases + +See the [full release notes for more improvements](https://github.com/elixir-lang/elixir/releases/tag/v1.10.0). + +## Improvements to sort-based APIs in Enum + +[`Enum.sort/1`](https://hexdocs.pm/elixir/Enum.html#sort/1) in Elixir by default sorts from lowest to highest: + +```elixir +iex> Enum.sort(["banana", "apple", "pineapple"]) +["apple", "banana", "pineapple"] +``` + +If you want to sort from highest to lowest, you need to call `Enum.sort/2` with a custom sorting function, such as `Enum.sort(collection, &>=/2)`, which is not immediately obvious to someone reading the code: + +```elixir +iex> Enum.sort(["banana", "apple", "pineapple"], &>=/2) +["pineapple", "banana", "apple"] +``` + +Furthermore, comparison operators, such as `<=` and `>=`, perform structural sorting, instead of a semantic one. For example, using `>=` to sort dates descendingly won't yield the correct result: + +```elixir +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]]) +[~D[2020-01-01], ~D[2019-12-31]] +``` + +To perform proper semantic comparison for dates, one would also need to pass a custom sorting function: + +```elixir +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]], &(Date.compare(&1, &2) != :lt)) +[~D[2019-12-31], ~D[2020-01-01]] +``` + +Elixir v1.10 streamlines the sorting functions by introducing both `:asc` and `:desc` shortcuts: + +```elixir +iex> Enum.sort(["banana", "apple", "pineapple"], :asc) +["apple", "banana", "pineapple"] +iex> Enum.sort(["banana", "apple", "pineapple"], :desc) +["pineapple", "banana", "apple"] +``` + +As well as adding the possibility to pass a module to perform semantic comparisons. For example, to sort dates, one now only needs to pass the `Date` module or even `{:desc, Date}` for descending semantical sort: + +```elixir +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]], Date) +[~D[2019-12-31], ~D[2020-01-01]] +iex> Enum.sort([~D[2019-12-31], ~D[2020-01-01]], {:desc, Date}) +[~D[2020-01-01], ~D[2019-12-31]] +``` + +These API improvements make the code more concise and readable and they have also been added to `Enum.sort_by`, `Enum.min_by`, `Enum.max_by`, and friends. + +### Tracking of compile-time configuration + +In Elixir, we organize our code in applications. Libraries, your dependencies, and your own project are all separate applications. All applications in Elixir also come with an application environment. + +The application environment is a key-value store that allows us to configure said application. While reading the application environment at runtime is the preferred approach, in some rare occasions you may want to use the application environment to configure the compilation of a certain project. This is often done by calling `Application.get_env/3` outside of a function: + +```elixir +defmodule MyApp.DBClient do + @db_host Application.get_env(:my_app, :db_host, "db.local") + + def start_link() do + SomeLib.DBClient.start_link(host: @db_host) + end +end +``` + +This approach has one big limitation: if you change the value of the application environment after the code is compiled, the value used at runtime is not going to change! For example, if you are using `mix release` and your `config/releases.exs` has: + + config :my_app, :db_host, "db.production" + +Because `config/releases.exs` is read after the code is compiled, the new value will have no effect as the code was compiled to connect to "db.local". + +Of course, the obvious solution to this mismatch is to not read the application environment at compilation time in the first place, and instead move the code to inside a function: + +```elixir +defmodule MyApp.DBClient do + def start_link() do + SomeLib.DBClient.start_link(host: db_host()) + end + + defp db_host() do + Application.get_env(:my_app, :db_host, "db.local") + end +end +``` + +While this is the preferred approach, there are still two scenarios we need to address: + + 1. Not everyone may be aware of this pitfall, so they will mistakenly read the application environment at compile-time, until they are bitten by this behaviour + + 2. In rare occasions, you truly need to read the application environment at compile-time, and you want to be warned when you try to configure at runtime something that is valid only at compilation time + +Elixir v1.10 aims to solve these two scenarios by introducing a `Application.compile_env/3` function. For example, to read the value at compile time, you can now do: + +```elixir +@db_host Application.compile_env(:my_app, :db_host, "db.local") +``` + +By using `compile_env/3`, Elixir will store the values used during compilation and compare them with the runtime values whenever your system starts, raising an error in case they differ. This helps developers ensure they are running their production systems with the configuration they intend to. + +In future versions, we will deprecate the use `Application.get_env/3` at compile-time with a clear message pointing users to configuration best practices, effectively addressing the scenario where users read from the application environment at compile time unaware of its pitfalls. + +### Compiler tracing + +This release brings enhancements to the Elixir compiler and adds new capabilities for developers to listen to compilation events. + +In previous Elixir versions, Elixir would compile a database of cross references between modules (such as function calls, references, structs, etc) for each project in order to perform all kinds of checks, such as deprecations and undefined functions. + +Although this database was not public, developers would still use it to run their own checks against their projects. With time, developers would request more data to be included in the database, which was problematic as Elixir itself did not have a use for the additional data, and the database was not meant to be used externally in the first place. + +In Elixir v1.10, we have addressed these problems by [introducing compiler tracing](https://hexdocs.pm/elixir/Code.html#module-compilation-tracers). The compiler tracing allows developers to listen to events as they are emitted by the compiler, so they can store all of the information they need - and only the information they need. + +Elixir itself is using the new compiler tracing to provide new functionality. One advantage of this approach is that developers can now disable undefined function warnings directly on the callsite. For example, imagine you have an optional dependency which may not be available in some cases. You can tell the compiler to skip warning on calls to optional modules with: + + @compile {:no_warn_undefined, OptionalDependency} + defdelegate my_function_call(arg), to: OptionalDependency + +Previously, this information had to be added to the overall project configuration, which was far away from where the optional call effectively happened. + +### Other enhancements + +Elixir's calendar data types got many improvements, such as sigil support for third-party calendars, as well as the additions of [`DateTime.now!/2`](https://hexdocs.pm/elixir/DateTime.html#now!/2), [`DateTime.shift_zone!/3`](https://hexdocs.pm/elixir/DateTime.html#shift_zone!/3), and [`NaiveDateTime.local_now/0`](https://hexdocs.pm/elixir/NaiveDateTime.html#local_now/0). + +There are many improvements related to Elixir's AST in this release too. [`Code.string_to_quoted/2`](https://hexdocs.pm/elixir/Code.html#string_to_quoted/2) has two new options, `:token_metadata` and `:literal_encoder`, that give more control over Elixir's parser. This information was already available to the Elixir code formatter and has now been made public. These changes alongside compiler tracing means tools like [Credo](https://github.com/rrrene/credo), [Boundary](https://github.com/sasa1977/boundary), and IDE integrations have an even better foundation to analyze the source code. + +[ExUnit](https://hexdocs.pm/ex_unit), our test framework, ships two small but important improvements: `ExUnit.CaptureIO` can now be used by tests that run concurrently and we have added "pattern-matching diffing". To understand the last feature, take this code: + +```elixir +assert %{"status" => 200, "body" => %{"key" => "foo"}} = json_payload +``` + +Now imagine that `json_payload` is a large JSON blob and the `"key"` inside the `"body"` did not have value of `"foo"`. In previous Elixir versions, if the assertion failed, Elixir would print the right side and let you up to your own devices to figure out what went wrong. In Elixir v1.10, we diff the data structure against the pattern so you can see exactly which parts of the data matched the pattern and which ones did not. Note ExUnit already performed diffing when comparing data types, this new version adds diffing when matching data against a pattern. + +Finally, this release also adds two new guards, `is_struct/1` and `is_map_key/2`, thanks to the strict requirement on Erlang/OTP 21+. + +To learn what else is new, you can read the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.10.0). + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2020-08-20-embedded-elixir-at-farmbot.markdown b/_posts/2020-08-20-embedded-elixir-at-farmbot.markdown new file mode 100644 index 000000000..0471d047f --- /dev/null +++ b/_posts/2020-08-20-embedded-elixir-at-farmbot.markdown @@ -0,0 +1,51 @@ +--- +layout: post +title: Embedded Elixir at Farmbot +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Farmbot. +logo: /images/cases/logos/farmbot.png +tags: embedded nerves +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[FarmBot](https://farm.bot/) is an open-source precision agriculture CNC farming project that includes a robot farming machine, software, and documentation including a farming data repository. FarmBot's machines use IoT technology to make it easy for farmers to remotely manage their gardens. + +Farmbot is built with [Nerves](https://www.nerves-project.org/), an open-source platform and infrastructure to build, deploy, and securely manage your fleet of IoT devices at speed and scale. + +When Connor Rigby, former embedded systems engineer at Farmbot, initially joined the company, his first project was to build a Farmbot application using Ruby. After completing the proof-of-concept, he knew that he needed a way to make the embedded development process more efficient, reliable, and secure. Connor had used Nerves before at a previous company and was a regular open-source contributor to Nerves, so he knew the platform would help him accomplish these goals. + +![Farmbot](/images/cases/bg/farmbot.jpg) + +## How Nerves helped + +Connor brought Nerves to Farmbot by porting the entire proof-of-concept Ruby application he'd created over to Nerves, which he did in his free time over the course of a month, taking him about 20 hours total. He also continued to make open-source contributions to Nerves, helping to structure the networking functionality that is now part of [NervesHub](https://www.nerves-hub.org/), the extensible web service that enables over-the-air firmware update management. + +
+

The biggest benefit of using Nerves is definitely how fast you can get up and running.

+

— Connor Rigby, Embedded Systems Engineer

+
+ +Connor says that the Nerves Platform and what eventually became NervesHub was a great choice for Farmbot because: + +### 1. Nerves supports lean systems and operates well in low-bandwidth areas + +Because Nerves bundles entire applications into relatively small archives in terms of firmware images for full Linux systems, Farmbot can use NervesHub to send over-the-air updates more quickly and users can download them faster. For comparison, an Android update generally clocks in at around 4 GB, but a Nerves update can be packed into as little as 12 MB. + +This is especially helpful for Farmbot users who operate in more remote locations with lower bandwidth and less reliable access to Wi-Fi. When an internet connection is available, NervesHub will connect and check if there's an update, and then prompt the user to install the update. + +### 2. Nerves adds convenience with low overhead + +For devices that are already connected to the internet, connecting to Nerves requires no additional configuration because NervesHub is compatible with the current public key infrastructure for device-to-cloud communication. Since Farmbot already had internet-connected devices when they brought Nerves onboard, they were able to use the same "key" to sign in to NervesHub that they use for their cloud service. + +### 3. Nerves has all the benefits of Elixir and Erlang + +Because it's written in Elixir and built within the Erlang runtime system, Nerves retains the qualities of that language and framework — notably that they are distributed, fault-tolerant, soft real-time, and highly available. Connor also says that with Nerves, it's easy to reason about the things you build with Nerves because you only input what you need into a Nerves application, helping you to avoid unnecessary complexities or unforeseen security vulnerabilities. You can check up on devices as they're running and debug them without disruption to the user experience. + +## The result + +FarmBot now has around 300 devices live in NervesHub, with a different deployment for each of their device models. Nerves is built to scale, so as Farmbot continues to grow its user base and expand their product capabilities, they'll be able to continue developing and releasing reliable firmware updates using Nerves. + +*This case study has first been published on [Nerves' website](https://nerves-project.org/cases/farmbot)*. diff --git a/_posts/2020-09-24-paas-with-elixir-at-Heroku.markdown b/_posts/2020-09-24-paas-with-elixir-at-Heroku.markdown new file mode 100644 index 000000000..bf97dd5f3 --- /dev/null +++ b/_posts/2020-09-24-paas-with-elixir-at-Heroku.markdown @@ -0,0 +1,69 @@ +--- +layout: post +title: PaaS with Elixir at Heroku +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Heroku. +logo: /images/cases/logos/heroku.png +tags: paas phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Heroku](https://www.heroku.com) provides services and tools to build, run, and scale web applications. They enable developers and teams to focus on the design and craft of their apps. Heroku started development back in 2007, focused on the Ruby programming language, and since then, they have expanded to support multiple runtimes, either officially or via buildpacks. + +As the platform grew, their engineering teams also adopted different languages, one of them being Elixir. In this article, we will talk about how two distinct engineering teams at Heroku, the Front-end team and the Vault team, have adopted Elixir. + +![Heroku](/images/cases/bg/heroku.png) + +## First steps with Elixir + +The Vault team was first to use Elixir inside Heroku. Their team is responsible for licensing and financial services, such as invoicing, credit card payments, etc. Most of their services are used internally at Heroku. + +They had to rewrite one of their existing services and that was the perfect occasion to give Elixir a try, since the difficulties and risks with the service were mostly known. The experiment was a success: they deployed and ran their first Elixir application in production. This paved the way to use Elixir more and more. + +Later on, they had a new challenge: they had to audit a large amount of data, and they knew from experience that the Ruby implementation would take too long to finish. Given they were already ramping up their familiarity with Elixir, they chose to apply [Elixir's GenStage](https://github.com/elixir-lang/gen_stage) to the problem, which is a low-level library for data processing, and that took only a couple hours. From this moment on, they were sold on the language and the platform. + +## Tackling operational complexity with Elixir + +The Front-end team shares a similar story: they first used Elixir to solve a well-understood problem and took it forward from there. + +The Front-end engineers are responsible for maintaining all user interfaces: the CLI, the dashboard, and a bunch of backend services that work with data. One of the features they provide to Heroku customers is analytics. + +At first, they were sending their analytics to Mixpanel. However, they had some issues fetching the data, due to cross-domain concerns, and they decided to replace Mixpanel by an in-house Elixir service. The service used [Plug](https://github.com/elixir-plug/plug), a library for building web applications, and had a single endpoint. + +
+

We were having a lot of fun and a lot of luck with it, so we kept doing it.

+

— Micah Woods, Lead Engineer, on migrating to Elixir.

+
+ +They later spent most of a year focused on operational stability, and during this period, they started rewriting part of their Node.js microservices into Elixir. Today they have migrated their numerous Node.js microservices into one main Elixir application with one auxiliary service for authentication. The fact that Elixir was capable of handling everything they threw at it alongside their experience with Erlang's stability - [Heroku's router uses Erlang](https://blog.heroku.com/erlang-in-anger) - allowed them to simplify their operations considerably. + +## Productivity and scalability + +The Front-end team has been using Elixir for two years. The team has 21 engineers: about 4 of them doing Elixir full-time, and 8 engineers altogether doing Elixir here and there. + +The first service that they built with Elixir, the analytics services, receives requests and puts them into an in-memory queue to be processed within the same VM. It handles about 3k to 4k requests per second. 99% of the response times stay within 0-1ms, occasionally 4ms. They use 3 Heroku dynos for fault-tolerance - of course, Heroku uses Heroku for their own infrastructure. + +The main Elixir application uses [the Phoenix web framework](https://phoenixframework.org/) to power the Heroku Dashboard, provide real-time functionality via WebSockets, and support other services. This application runs on 5 Heroku dynos - although their engineering team believes they could probably do with less. Memory consumption is also on the lower side: their biggest dyno uses 256MB. + +The Vault team doing Elixir is only three engineers. Most of their apps are used internally, so they are generally not worried about performance. They continue using Elixir because they feel productive and happy with it. They have also found it is an easier language to maintain compared to their previous experiences. + +## On Phoenix + +Both teams generally use Phoenix for web applications, unless they have a reason not to, which is rare. They acknowledge there is not a performance penalty for using Phoenix and you get a lot out of the box. Phoenix makes it easy to opt-in on the pieces that they need and remove the parts that they don't want. + +They have also found it easier to understand how Phoenix itself works under the hood, especially compared to their previous experiences with other frameworks, such as Ruby on Rails. This knowledge is consistently helping them maintain and update their applications as time passes. + +## Learning Elixir and growing the team + +The growth of both Elixir teams has been mostly organic. Given there are multiple languages in their stack, they often hire for one or another language in particular, and not specifically for Elixir. If the new team members gravitate towards Elixir, they are further encouraged to explore and learn the language. They are also active practitioners of pair programming, so there are many opportunities in their team to learn from each other, rotate pairs, swap projects, and so on. + +According to Matthew Peck, "the paradigm shift from Object-Oriented languages to Functional Programming was our biggest challenge when first learning Elixir". However, the team agrees the investment was worth it: "Learning Elixir has made us better programmers. We have found that immutability made our code more readable, easier to test, and simpler to make concurrent. Now when we go back to an Object-Oriented language, we are thinking about how we can apply the same concepts there" - said Mike Hagerdon. + +Amanda Dolan added some remarks on Elixir's capabilities for writing concurrent and fault-tolerant applications: "One other challenge when learning Elixir is fully grasping concurrency and the Erlang/OTP patterns". Some of them felt it took longer to master those concepts than they first expected. + +Taylor Mock has his take on the challenges teams may face when adopting Elixir: "Another difference between Elixir and our previous stacks, Ruby and Node.js, is in the ecosystems". They were initially concerned that the Elixir ecosystem would lack when it comes to third-party tools, but that was not what they saw. Taylor continues: "We found out that we can get really far with the concepts and mechanisms that the language itself provides. This shift can be scary, but we are now past it, and we find ourselves with leaner applications and fewer dependencies". + +Overall, both teams found the language itself quite approachable. Given they started with a small proof of concept, they were able to tackle their concerns in regards to adoption, development, and deployment as they moved forward. Historically Heroku also has had much success with Erlang, and that has contributed to the success adopting Elixir has seen inside Heroku. diff --git a/_posts/2020-10-06-elixir-v1-11-0-released.markdown b/_posts/2020-10-06-elixir-v1-11-0-released.markdown new file mode 100644 index 000000000..7aaaefefb --- /dev/null +++ b/_posts/2020-10-06-elixir-v1-11-0-released.markdown @@ -0,0 +1,225 @@ +--- +layout: post +title: Elixir v1.11 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.11 is out with improvements to the compiler and tighter integration with Erlang/OTP +--- + +Over the last releases, the Elixir team has been focusing on the compiler, both in terms of catching more mistakes at compilation time and making it faster. Elixir v1.11 has made excellent progress on both fronts. This release also includes many other goodies, such as tighter Erlang integration, support for more guard expressions, built-in datetime formatting, and other calendar enhancements. + +During this period, we have also started [publishing a series of production cases](/cases.html) on our website, featuring Elixir's usage at [Farmbot](/blog/2020/08/20/embedded-elixir-at-farmbot/) and [Heroku](/blog/2020/09/24/paas-with-elixir-at-Heroku/), with many more cases coming soon. + +For now, let's focus on what's new in Elixir v1.11. + +## Tighter Erlang integration + +On the footsteps of v1.10, we have further integrated with Erlang's new logger by adding four new log levels: `notice`, `critical`, `alert`, and `emergency`, matching all log levels found in the Syslog standard. The `Logger` module now supports structured logging by passing maps and keyword lists to its various functions. It is also possible to specify the log level per module, via the [`Logger.put_module_level/2`](https://hexdocs.pm/logger/Logger.html#put_module_level/2) function. Log levels per application will be added in future releases. + +IEx also has been improved to show the documentation for Erlang modules directly from your Elixir terminal. For example, here is a clip of me accessing the documentation for [Erlang's gen_server module](erlang.org/doc/man/gen_server.html): + + + + This works with Erlang/OTP 23+ and requires Erlang modules to have been compiled with documentation chunks. A huge thank you to the Erlang/OTP team and the Documentation Working Group of the [Erlang Ecosystem Foundation](https://erlef.org/) for making this possible. + +## Compiler checks: application boundaries + +Elixir v1.11 builds on top of the recently added compilation tracers to track application boundaries. From this release, Elixir will warn if you invoke a function from an existing module but this module does not belong to any of your listed dependencies. + +These two conditions may seem contradictory. After all, if a module is available, it must have come from a dependency. This is not true in two scenarios: + + * Modules from Elixir and Erlang/OTP are always available - even if their applications are not listed as a dependency + + * In an umbrella project, because all child applications are compiled within the same VM, you may have a module from a sibling project available, even if you don't depend on said sibling + +This new compiler check makes sure that all modules that you invoke are listed as part of your dependencies, emitting a warning like below otherwise: + +```text +:ssl.connect/2 defined in application :ssl is used by the current +application but the current application does not directly depend +on :ssl. To fix this, you must do one of: + + 1. If :ssl is part of Erlang/Elixir, you must include it under + :extra_applications inside "def application" in your mix.exs + + 2. If :ssl is a dependency, make sure it is listed under "def deps" + in your mix.exs + + 3. In case you don't want to add a requirement to :ssl, you may + optionally skip this warning by adding [xref: [exclude: :ssl] + to your "def project" in mix.exs +``` + +This comes with extra benefits in umbrella projects, as it requires applications to depend on the siblings they depend on, which will fail if there are any cyclic dependencies. + +## Compiler checks: data constructors + +In Elixir v1.11, the compiler also tracks structs and maps fields across a function body. For example, imagine you wanted to write this code: + +```elixir +def drive?(%User{age: age}), do: age >= 18 +``` + +If there is either a typo on the `:age` field or the `:age` field was not yet defined, the compiler will fail accordingly. However, if you wrote this code: + +```elixir +def drive?(%User{} = user), do: user.age >= 18 +``` + +The compiler would not catch the missing field and an error would only be raised at runtime. With v1.11, Elixir will track the usage of all maps and struct fields within the same function, emitting warnings for cases like above: + +```text +warning: undefined field `age` in expression: + + # example.exs:7 + user.age + +expected one of the following fields: name, address + +where "user" was given the type %User{} in: + + # example.exs:7 + %User{} = user + +Conflict found at + example.exs:7: Check.drive?/1 +``` + +The compiler also checks binary constructors. Consider you have to send a string over the wire with length-based encoding, where the string is prefixed by its length, up to 4MBs. Your initial attempt may be this: + +```elixir +def run_length(string) when is_binary(string) do + <> +end +``` + +However, the code above has a bug. Each segment given between `<<>>` must be an integer, unless specified otherwise. With Elixir v1.11, the compiler will let you know so: + +```text +warning: incompatible types: + + binary() !~ integer() + +in expression: + + <> + +where "string" was given the type integer() in: + + # foo.exs:4 + <> + +where "string" was given the type binary() in: + + # foo.exs:3 + is_binary(string) + +HINT: all expressions given to binaries are assumed to be of type integer() +unless said otherwise. For example, <> assumes "expr" is an integer. +Pass a modifier, such as <> or <>, to change the +default behaviour. + +Conflict found at + foo.exs:4: Check.run_length/1 +``` + +Which can be fixed by adding `::binary` to the second component: + +```elixir +def run_length(string) when is_binary(string) do + <> +end +``` + +While some of those warnings could be automatically fixed by the compiler, future versions will also perform those checks across functions and potentially across modules, where automatic fixes wouldn't be desired (nor possible). + +## Compilation time improvements + +Elixir v1.11 features many improvements to how the compiler tracks file dependencies, such that touching one file causes less files to be recompiled. In previous versions, Elixir tracked three types of dependencies: + + * compile time dependencies - if A depends on B at compile time, such as by using a macro, whenever B changes, A is recompiled + * struct dependencies - if A depends on B's struct, whenever B's struct definition changed, A is recompiled + * runtime dependencies - if A depends on B at runtime, A is never recompiled + +However, because dependencies are transitive, if A depends on B at compile time and B depends on C at runtime, A would depend on C at compile time. Therefore, it is very important to reduce the amount of compile time dependencies. + +Elixir v1.11 replaces "struct dependencies" by "exports dependencies". In other words, if A depends on B, whenever B public's interface changes, A is recompiled. B's public interface is made by its struct definition and all of its public functions and macros. + +This change allows us to mark `import`s and `require`s as "exports dependencies" instead of "compile time" dependencies. This simplifies the dependency graph considerably. For example, [in the Hex.pm project](https://github.com/hexpm/hexpm), changing the `user.ex` file in Elixir v1.10 would emit this: + +```text +$ touch lib/hexpm/accounts/user.ex && mix compile +Compiling 90 files (.ex) +``` + +In Elixir v1.11, we now get: + +```text +$ touch lib/hexpm/accounts/user.ex && mix compile +Compiling 16 files (.ex) +``` + +To make things even better, Elixir v1.11 also introduces a more granular tracking for umbrella projects (and path dependencies in general). In previous versions, a module from a sibling application would always be treated as a compile time dependency. This often meant that changing an application would cause many modules in sibling applications to recompile. Elixir v1.11 will tag modules from dependencies as exports whenever possible, yielding dramatic improvements in those cases. + +To round up the list of compiler enhancements, the `--profile=time` option added in Elixir v1.10 now also includes the time to compile each individual file. For example, in the Plug project, one can now get: + +```text +[profile] lib/plug/conn.ex compiled in 935ms +[profile] lib/plug/ssl.ex compiled in 147ms (plus 744ms waiting) +[profile] lib/plug/static.ex compiled in 238ms (plus 654ms waiting) +[profile] lib/plug/csrf_protection.ex compiled in 237ms (plus 790ms waiting) +[profile] lib/plug/debugger.ex compiled in 719ms (plus 947ms waiting) +[profile] Finished compilation cycle of 60 modules in 1802ms +[profile] Finished group pass check of 60 modules in 75ms +``` + +While implementing those features, we have also made the `--long-compilation-threshold` flag more precise. In previous versions, `--long-compilation-threshold` would consider both the time a file spent to compile and the time spent waiting on other files. In Elixir v1.11, it considers only the compilation time. This means less false positives and you can now effectively get all files that take longer than 2s to compile, in execution time, by passing `--long-compilation-threshold 2`. + +## `config/runtime.exs` and `mix app.config` + +Elixir v1.9 introduced a new configuration file called `config/releases.exs`. However, this new configuration file was executed only during releases. For those not familiar with releases, a release is a self-contained artifact with the Erlang VM, Elixir and your application, ready to run in production. + +This new configuration file was considered a very useful addition to releases. Therefore, we are also introducing `config/runtime.exs`, which is executed after the code compilation on all environments (dev, test, and prod) - for both Mix and releases. Our goal is to provide a better runtime configuration experience to developers, in contrast to our current configuration system which has been mostly compile-time centric. + +`config/runtime.exs` works the same as any other configuration file in Elixir. However, given `config/runtime.exs` is meant to run in production systems, where our `Mix` build tool is not available, developers must not use [`Mix.env()`](https://hexdocs.pm/mix/Mix.html#env/0) or [`Mix.target()`](https://hexdocs.pm/mix/Mix.html#target/0) in `config/runtime.exs`. Instead, they must use the new `config_env()` and `config_target()`, which have been added to the [`Config`](https://hexdocs.pm/elixir/Config.html) module. + +While `config/releases.exs` will continue to be supported, developers can migrate to `config/runtime.exs` without loss of functionality. For example, a `config/releases.exs` file such as this one + +```elixir +# config/releases.exs +import Config + +config :foo, ... +config :bar, ... +``` + +could run as is as `config/runtime.exs`. However, given `config/runtime.exs` runs in all environments, you may want to restrict part of your configuration to the `:prod` environment: + +```elixir +# config/runtime.exs +import Config + +if config_env() == :prod do + config :foo, ... + config :bar, ... +end +``` + +If both files are available, releases will pick the now preferred `config/runtime.exs` instead of `config/releases.exs`. + +To wrap it all up, `Mix` also includes a new task called [`mix app.config`](https://hexdocs.pm/mix/Mix.Tasks.App.Config.html). This task loads all applications and configures them, without starting them. Whenever you write your own Mix tasks, you will typically want to invoke either `mix app.start` or `mix app.config` before running your own code. Which one is better depends if you want your applications running or only configured. + +## Other improvements + +Elixir v1.11 adds the `is_struct/2`, `is_exception/1`, and `is_exception/2` guards. It also adds support for the `map.field` syntax in guards. + +The Calendar module ships with a new [`Calendar.strftime/3`](https://hexdocs.pm/elixir/Calendar.html#strftime/3) function, which provides datetime formatting based on the `strftime` format. The [`Date`](https://hexdocs.pm/elixir/Date.html) module got new functions for working with weeks and months, such as `Date.beginning_of_month/1` and `Date.end_of_week/2`. Finally, all calendar types got conversion functions from and to gregorian timestamps, such as `Date.from_gregorian_days/2` and `NaiveDateTime.to_gregorian_seconds/1`. + +Finally, to bring visibility to the compiler tracking improvements described in previous sections, we have also added new features to [`mix xref`](https://hexdocs.pm/mix/Mix.Tasks.Xref.html). `mix xref` is a task that describes cross-references between files in your projects and can be helpful to diagnose large compilation cycles in projects. + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.11.0). + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2020-10-08-real-time-communication-at-scale-with-elixir-at-discord.markdown b/_posts/2020-10-08-real-time-communication-at-scale-with-elixir-at-discord.markdown new file mode 100644 index 000000000..ac4f42ef8 --- /dev/null +++ b/_posts/2020-10-08-real-time-communication-at-scale-with-elixir-at-discord.markdown @@ -0,0 +1,70 @@ +--- +layout: post +title: Real time communication at scale with Elixir at Discord +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Discord. +flagship: true +logo: /images/cases/logos/discord.png +tags: real-time genstage otp +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +Founded in 2015 by Jason Citron and Stan Vishnevskiy, [Discord](https://discord.com/) is a permanent, invite-only space for your communities and friends, where people can hop between voice, video, and text, depending on how they want to talk, letting them have conversations in a very natural or authentic way. Today, the service has over 100 million monthly active users from across the globe. Every day people spend 4 billion minutes in conversation on Discord servers, across 6.7 million active servers / communities. + +From day one, Discord has used Elixir as the backbone of its chat infrastructure. When Discord first adopted the language, they were still working on building a viable business, with many questions and challenges in front of them. Elixir played a crucial role in giving them the desired technological flexibility to grow the company and also became the building block that would allow their systems to run on a massive scale. + +![Discord](/images/cases/bg/discord.jpg) + +## Starting technologies + +Back in 2015, Discord chose two main languages to build their infrastructure: Elixir and Python. Elixir was initially picked to power the WebSocket gateway, responsible for relaying messages and real-time replication, while Python powered their API. + +Nowadays, the Python API is a monolith while the Elixir stack contains 20 or so different services. These architectural choices do not represent a dichotomy between the languages but rather a pragmatic decision. Mark Smith, from the Discord team, explains it succinctly: "given the Elixir services would handle much bigger traffic, we designed them in a way where we could scale each service individually." + +Discord has also explored other technologies along the way, Go and Rust being two examples, with distinct outcomes. While Discord completely phased out Go after a short foray, Rust has proven to be an excellent addition to their toolbox, boosted by its ability to play well with Elixir and Python. + +## Communication at scale + +Effective communication plays an essential role when handling millions of connected users concurrently. To put things into perspective, some of Discord's most popular servers, such as those dedicated to Fortnite and Minecraft, are nearing six hundred thousand users. At a given moment, it is not unlikely to encounter more than two hundred thousand active users in those servers. If someone changes their username, Discord has to broadcast this change to all connected users. + +Overall, Discord's communication runs at impressive numbers. They have crossed more than 12 million concurrent users across all servers, with more than 26 million WebSocket events to clients per second, and Elixir is powering all of this. + +
+

In terms of real time communication, the Erlang VM is the best tool for the job.

+

— Jake Heinz, Lead Software Engineer

+
+ +When we asked their team "Why Elixir?", Jake Heinz gave a straight-forward answer: "In terms of real time communication, the Erlang VM is the best tool for the job. It is a very versatile runtime with excellent tooling and reasoning for building distributed systems". Technologically speaking, the language was a natural fit. However, Elixir was still a bet back in 2015: "Elixir v1.0 had just come out, so we were unsure in which direction the language would go. Luckily for us, we have been pleased with how the language has evolved and how the community shaped up." + +## The chat infrastructure team + +To power their chat messaging systems, Discord runs a cluster with 400-500 Elixir machines. Perhaps, the most impressive feat is that Discord's chat infrastructure team comprises five engineers. That's right: five engineers are responsible for 20+ Elixir services capable of handling millions of concurrent users and pushing dozens of millions of messages per second. + +Discord also uses Elixir as the control plane of their audio and video services, also known as signaling, which establishes communication between users. C++ is then responsible for media streaming, a combination that altogether runs on 1000+ nodes. + +The Elixir services communicate between them using Distributed Erlang, the communication protocol that ships as part of the Erlang Virtual Machine. By default, Distributed Erlang builds a fully meshed network, but you can also ask the Erlang VM to leave the job of outlining the topology up to you, by setting the aptly named `-connect_all false` flag. The Discord team sets this option to assemble a partially meshed network with [etcd](https://etcd.io/) being responsible for service discovery and hosting shared configuration. + +The chat infrastructure developers are not the only ones touching the Elixir codebases. According to Mark Smith, this is an important part of Discord's culture: "We don't work in silos. So a Python developer may have to work on the Elixir services when building a new feature. We will spec out the feature together, figure out the scalability requirements, and then they will work on a pull request, which we will review and help them iterate on it." + +## Community and challenges + +To run at this scale, Discord learned how to leverage the Erlang VM's power, its community, and when to recognize challenges that require them to reach for their own solutions. + +For example, Discord uses [Cowboy](https://github.com/ninenines/cowboy/) for handling WebSocket connections and TCP servers. To manage data bursts and provide load regulation, such as back-pressure and load-shedding, they use [GenStage](https://github.com/elixir-lang/gen_stage), which they have [discussed in detail in the past](https://discord.com/blog/how-discord-handles-push-request-bursts-of-over-a-million-per-minute-with-elixirs-genstage). + +Other times, the efforts of the company and the community go hand in hand. That was the case when Discord used [the Rustler project](https://github.com/rusterlium/rustler), which provides a safe bridge between Elixir and Rust, to [scale to 11 million concurrent users](https://discord.com/blog/using-rust-to-scale-elixir-for-11-million-concurrent-users). They used the Rustler to hook a custom data structure built in Rust directly into their Elixir services. + +However, the team has made abundantly clear that the powerhouse is the Erlang platform. Every time they had to push their stack forward, they never felt cornered by the technology. Quite the opposite, their engineers could always build efficient solutions that run at Discord's scale, often in a few hundred lines of code. Discord frequently gives these projects back to the community, as seen in [Manifold](https://github.com/discord/manifold) and [ZenMonitor](https://github.com/discord/zen_monitor). + +The Discord team also adapted quickly when things went wrong. For instance, they attempted twice to use [Mnesia](https://www.erlang.org/doc/man/mnesia.html) in production —a database that ships as part of Erlang's standard library. They tried Mnesia in persistent and in-memory modes, and the database nodes would often fall behind in failure scenarios, sometimes being unable to ever catch up. Eventually they ditched Mnesia altogether and built the desired functionality with Erlang's builtin constructs, such as GenServer and ETS. Nowadays, they resolve these same failure scenarios within 2-3 seconds. + +## Mastering Elixir + +None of the chat infrastructure engineers had experience with Elixir before joining the company. They all learned it on the job. Team members Matt Nowack and Daisy Zhou report initially struggling to understand how all of their services communicate. Matt adds: "In the beginning, it was hard to accept all of the guarantees that Erlang VM provides. I'd worry about data races and concurrency issues that were impossible to happen". Eventually, they took these guarantees to heart and found themselves more productive and more capable of relying on the platform and its tools. Matt continues: "The introspection tools the Erlang VM provides is the best in class. We can look at any VM process in the cluster and see its message queue. We can use the remote shell to connect to any node and debug a live system. All of this has helped us countless times." + +Running at Discord's scale adds its own dimension to mastering the language, as they need to familiarize with the abstractions for providing concurrency, distribution, and fault-tolerance. Nowadays, frameworks such as Nerves and Phoenix handle these concerns for developers, but the underlying building blocks are always available for engineers assembling their own stack, such as the Discord team. + +In the end, Jake summarized how crucial Elixir and the Erlang VM have been at Discord and how it affected him personally: "What we do in Discord would not be possible without Elixir. It wouldn't be possible in Node or Python. We would not be able to build this with five engineers if it was a C++ codebase. Learning Elixir fundamentally changed the way I think and reason about software. It gave me new insights and new ways of tackling problems." diff --git a/_posts/2020-10-27-delivering-social-change-with-elixir-at-change.org.markdown b/_posts/2020-10-27-delivering-social-change-with-elixir-at-change.org.markdown new file mode 100644 index 000000000..8e58ec64c --- /dev/null +++ b/_posts/2020-10-27-delivering-social-change-with-elixir-at-change.org.markdown @@ -0,0 +1,85 @@ +--- +layout: post +title: Delivering social change with Elixir at Change.org +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Change.org. +flagship: true +logo: /images/cases/logos/change.png +tags: social broadway +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Change.org](https://change.org/) is a social change platform, with over 400 million users worldwide. Two years ago, their engineering team faced a challenge to migrate their messaging system from an external vendor to an in-house solution, to reduce costs and gain flexibility. + +This article will discuss how they approached this problem, why they chose Elixir, and how their system grew to deliver more than 1 billion emails per month. Change.org is also [hiring Elixir engineers to join their team](https://www.change.org/careers). + +![Change.org](/images/cases/bg/change.png) + +## The path to Elixir + +The first step for Change.org's engineering team was to outline the requirements for their system. The system would receive millions of events, such as campaign updates, new petitions, and more, and it should send emails to all interested parties whenever appropriate. They were looking for an event-driven solution at its core, in which concurrency and fault-tolerance were strong requirements. + +The next stage was to build proofs-of-concept in different programming languages. Not many companies can afford this step, but Change.org's team knew the new system was vital to their business and wanted to be thorough in their analysis. + +Around this time, John Mertens, Director of Engineering, was coming back from parental leave. He used this opportunity to catch up with different technologies whenever possible. That's when he stumbled upon [José Valim's presentation at Lambda Days](https://www.youtube.com/watch?v=XPlXNUXmcgE), which discussed two libraries in the Elixir ecosystem: [GenStage](https://github.com/elixir-lang/gen_stage) and [Flow](https://github.com/dashbitco/flow). + +They developed prototypes in four technologies: JRuby, Akka Streams, Node.js, and Elixir. The goal was to evaluate performance, developer experience, and community support for their specific use cases. Each technology had to process 100k messages as fast as possible. John was responsible for the Elixir implementation and put his newly acquired knowledge to use. + +After two evaluation rounds, the team chose to go ahead with Elixir. Their team of 3 engineers had 18 months to replace the stack they had been using for the last several years with their own Elixir implementation. + +## Learning Elixir + +When they started the project, none of the original team members had prior experience with Elixir. Only Justin Almeida, who joined when the project had been running by six months, had used Elixir before. + +Luckily, the team felt supported by the different resources available in the community. John recalls: "We were in one of our early meetings discussing how to introduce Elixir into our stack when Pragmatic Programmers announced the [Adopting Elixir](https://pragprog.com/titles/tvmelixir/adopting-elixir/) book, which was extremely helpful in answering many of our questions." + +## The new system + +The team developed three Elixir applications to replace the external vendor. The first application processes all incoming events to decide whether an email should go out and to whom. + +The next application is the one effectively responsible for dispatching the emails. For each message, it finds the appropriate template as well as the user locale and preferences. It then assembles the email and delivers it with the help of a Mail Transfer Agent (MTA). + +The last application is responsible for analytics. It receives webhook calls from the MTA with batches of different events, which are processed and funneled into their data warehouse for later use. + +After about four months, they put the new system in production. While Change.org has dozens of different email templates, the initial deployment handled a single and straight-forward case: password recovery. + +Once the new system was in production, they continued to migrate different use cases to the system, increasing the numbers of handled events and delivered emails day after day. After one year, they had completed the migration ahead of schedule. + +## Handling spikes and load regulation + +Today, those applications run on a relatively small number of nodes. The first two applications use 6 to 8 nodes, while the last one uses only two nodes. + +John explains they are over-provisioned because spikes are relatively frequent in the system: "for large campaigns, a single event may fan out to thousands or hundreds of thousands of emails." + +The team was kind enough to share some of their internal graphs. In the example below, you can see a spike of over 10 million messages coming to the system: + +![Usage at Change.org](/images/cases/bg/change-graph.png) + +Once this burst happens, all nodes max their CPUs, emitting around 3000 emails per second until they drain the message queue. The whole time memory usage remains at 5%. + +The back-pressure provided by the [GenStage](https://github.com/elixir-lang/gen_stage) library played a crucial role in the system’s performance. Since those applications fetch events from message queues, process them, and submit them into third-party services, they must avoid overloading any part of the stack. GenStage addresses this by allowing the different components, called stages in the library terminology, to communicate how much data they can handle right now. For example, if sending messages to the MTA is slower than usual, the system will naturally get fewer events from the queue. + +Another essential feature of the system is to work in batches. Receiving and sending data is more efficient and cost-effective if you can do it in groups instead of one-by-one. John has given [a presentation at ElixirConf Europe sharing the lessons learned from their first trillion messages](https://www.youtube.com/watch?v=t46L9RKmlNo). + +The activity on Change.org has grown considerably over the last year too. The systems have coped just fine. Justin remarks: "everything has been working so well that some of those services are not really on our minds." + +## Working with the ecosystem + +Change.org has relied on and contributed to the ecosystem whenever possible. During the migration, both old and new systems had to access many shared resources, such as [HAML templates](https://haml.info/), Ruby's I18N configuration files, and [Sidekiq's background queues](https://sidekiq.org/). Fortunately, they were able to find compatible libraries in the Elixir ecosystem, respectively [calliope](https://github.com/nurugger07/calliope), [linguist](https://github.com/change/linguist), and [exq](https://github.com/akira/exq). + +Nowadays, some of those libraries have fallen out of flavor. For example, the community has chosen gettext for internationalization, as it is a more widely accepted format. For this reason, Change.org has stepped in and taken ownership of the linguist library. + +As Change.org adopted Elixir, the ecosystem grew to better support their use cases too. One recent example [is the Broadway library](https://github.com/dashbitco/broadway), which makes it easy to assemble data pipelines. John explains: "Broadway builds on top of GenStage, so it provides the load regulation, concurrency, and fault-tolerance that we need. It also provides batching and partitioning, which we originally had to build ourselves. For new projects, Broadway is our first choice for data ingestion and data processing." + +## Elixir as the default stack + +As projects migrate to Elixir, Elixir has informally become the default stack at Change.org for backend services. Today they have more than twenty projects. The engineering team has also converged on a common pattern for services in their event driven architecture, built with Broadway and Phoenix. + +In a nutshell, they use Broadway to ingest, aggregate, and store events in the database. Then they use Phoenix to expose this data, either through APIs, as analytics or as tooling for their internal teams. + +One recent example is [Change.org's Bandit service](https://medium.com/making-change-org/our-elixir-bandit-service-e2b6af6eebc4). The service provides a Phoenix API that decides which copy to present to users in various parts of their product. As users interact with these copies, data is fed into the system and analyzed in batches with Broadway. They use this feedback to optimize and make better choices in the future. + +The team has also grown to ten Elixir developers thanks to the multiple training and communities of practice they have organized internally. Change.org is also looking for Elixir backend engineers, as they aim to bring experience and diversity to their group. Interested developers can [learn more about these opportunities on their website](https://www.change.org/careers). diff --git a/_posts/2020-11-17-real-time-collaboration-with-elixir-at-slab.markdown b/_posts/2020-11-17-real-time-collaboration-with-elixir-at-slab.markdown new file mode 100644 index 000000000..e699b459a --- /dev/null +++ b/_posts/2020-11-17-real-time-collaboration-with-elixir-at-slab.markdown @@ -0,0 +1,70 @@ +--- +layout: post +title: Real-time collaboration with Elixir at Slab +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Slab. +logo: /images/cases/logos/slab.png +tags: collab phoenix otp +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Slab](https://slab.com/) is a knowledge base and team wiki that democratizes knowledge. Jason Chen started Slab in August 2016, after picking Elixir and Phoenix as the best tools to build real-time collaborative applications. The company has grown to 6 engineers since then, distributed worldwide, and relied upon by more than 7000 companies and customers like Asana, Discord, and Glossier. If you are interested in helping companies become a source of learning and purpose, especially during these times where remote collaboration is essential, [Slab is hiring](https://slab.com/jobs?ref=elixir). + +![Slab](/images/cases/bg/slab.png) + +## Why Elixir? + +Slab was not the first time Jason wrote a collaborative web app. He had previous practice building them in Rails and Node.js and he believed there was a lot to improve in the development experience, especially when it came to working with WebSockets. Both technologies were also troublesome in production, as the team faced issues scaling them vertically and horizontally. + +
+

I wanted a framework with the same developer experience as Django and Rails, but one that was designed for real-time applications.

+

— Jason Chen, CEO, on the Phoenix web framework

+
+ +Jason doesn't consider himself a person who is always looking for new things, but he knew he would have to survey the options around him when starting Slab. During this period, he explored two main languages: Go and Elixir. In the end, Jason chose Elixir, thanks to [the Phoenix web framework](https://phoenixframework.org/): "I was looking for a framework that offered a complete toolset for building web apps. I was not interested in making low-level choices, such as which ORM to use, which library to pick for parsing requests, etc. I wanted a framework with the same developer experience as Django and Rails, but one that was designed for real-time applications". + +Jason gave himself two weeks to build a proof of concept. He wrote a collaborative blog, where multiple users could write a post simultaneously, and comments were added in real-time — all while learning Elixir and the Phoenix framework. + +The trial went better than expected, and Jason's journey with Slab had officially begun. + +## Growing with the platform + +Shortly after, Slab was in a private beta with a handful of companies as users. For each major feature they had along the way, Elixir and Phoenix provided the building blocks for it. When they implemented real-time comments, they used Phoenix Channels and Phoenix PubSub. The pattern goes on: "for asynchronous processing, we simply use [Elixir tasks](https://hexdocs.pm/elixir/Task.html)". Later on, to track users editing a document and give each a different cursor color, they used [Phoenix Presence](https://hexdocs.pm/phoenix/Phoenix.Presence.html), a tool that no other web framework offers out-of-the-box. + +Another leap in Jason's journey with Slab and Elixir was when he had to learn Erlang/OTP, a group of behaviors that ship as part of Erlang's standard library for building distributed and fault-tolerant applications. + +To improve the real-time collaborative editor that is part of Slab, Jason implemented [Operational Transformation](https://en.wikipedia.org/wiki/Operational_transformation). The client runs in the browser, implemented with [React](https://reactjs.org/). As users make changes to the text, their diffs are sent to the server, which arbitrates these updates and synchronizes them across the various clients. + +Tackling the synchronization problem is not trivial, especially when the application is running on multiple nodes. Here is the challenge they faced. Imagine user Alice has a WebSocket connection to node X and user Bob is connected to node Y. Both Alice and Bob are working on the same text. How can Slab guarantee that changes from both users are applied, so both see the same document once done editing? + +One could try to solve this problem by keeping the server stateless. Every time the server receives a diff from the client, the server would read the document from the database, apply the changes, normalize the result, and broadcast the clients' updates. With this approach, the issue is that loading the text from the database on every client update would quickly become expensive, especially as they grow in size. Response times would become higher and the user experience would degrade. + +When working with Node.js, Jason tried a different approach. If Alice and Bob were writing to the same document, a load balancer would guarantee that both would be routed to the same node. After trying out both Apache and Nginx, he implemented the balancer in Node.js. The overall solution was time-consuming to get right and introduced operational complexities. + +Luckily, these problems are the bread and butter of Erlang/OTP. Jason knew he needed a stateful abstraction to keep this state on the server. He had already heard about the options the platform provides, but he was unsure which one to pick. Jason recalls: "I remember asking the community if I should use an [Agent](https://hexdocs.pm/elixir/Agent.html) or a [GenServer](https://hexdocs.pm/elixir/GenServer.html) and everyone was really helpful in providing guidance." They quickly landed on GenServer as their tool of choice. + +By default, both GenServer and Agents are local to each node. However, they also support the `:global` option, which registers a given name across the cluster. To use this option, they need the Erlang distribution, which they were already using for Phoenix PubSub and Presence, so this was a straight-forward change. This guarantees both Alice and Bob talk to the same GenServer, regardless if they joined node X or node Y. + +Later on, when running the system in production, the platform continued to impress him. Every time they increased the machine resources, they could see the runtime efficiently using everything it had available, without changes to the code. + +## Learning and tools + +There are other few notable tools in Slab's stack. + +Back in 2017, they migrated to GraphQL [powered by Elixir's Absinthe](http://absinthe-graphql.org/). There were concerns about adopting the query language, as it was a relatively new technology. Still, they felt it would address a real issue: they had different components in the application needing distinct data, and managing all of these possible combinations was becoming complex. This was one of the main problems GraphQL was designed to solve. + +They are also running on Google Cloud with Kubernetes (K8s), and, as many Elixir engineers, they wondered [how the Erlang VM fit in a world with Docker and K8s](https://dashbit.co/blog/kubernetes-and-the-erlang-vm-orchestration-on-the-large-and-the-small). Today they run on 6 nodes, 5 of them running application code. The sixth one handles [cron jobs](https://en.wikipedia.org/wiki/Cron) and stays on standby for new deployments. They use [the peerage library](https://github.com/mrluc/peerage) to establish Distributed Erlang connections between the nodes. + +
+

We really value Elixir's ability to build complex systems using fewer moving parts. The code is simpler, and the system is easier to operate.

+

— Sheharyar Naseer, engineer

+
+ +Overall the Slab team aims to keep the number of dependencies low, something they believe is made possible by the platform and positively impacts onboarding new developers. Sheharyar Naseer, a member of their engineering team, explains: "We really value Elixir's ability to build complex systems using fewer moving parts. The code is simpler, and the system is easier to operate, making both experienced and new engineers more productive. We ran in production for more than 3 years without resorting to Redis. We just recently added it because we wanted our caches to survive across deployments. Many other stacks impose technologies like Redis from day one." + +This approach also yields benefits when updating libraries. Sheharyar continues: "For the most part, upgrading Erlang, Elixir, and Phoenix is straight-forward. We go through the CHANGELOG, which always emphasizes the main changes we need to perform, and we have a pull request ready after one or two hours. The only time we could not upgrade immediately was when Erlang/OTP removed old SSL ciphers, which broke our HTTP client and we caught it early on during development." + +When onboarding engineers, Slab recommends them different books and video courses — many of which you can find [in our learning resources page](/learning.html) — so they have the flexibility to choose a medium they are most productive with. New engineers also work on Slab itself and receive guidance through pull requests. They start with small tasks, usually in the client and GraphQL layers, and slowly tackle more complex problems around the database and Erlang/OTP. If you are interested in improving remote collaboration, [learn more about their opportunities on their website](https://slab.com/jobs?ref=elixir). diff --git a/_posts/2020-12-10-integrating-travel-with-elixir-at-duffel.markdown b/_posts/2020-12-10-integrating-travel-with-elixir-at-duffel.markdown new file mode 100644 index 000000000..44f8c66cb --- /dev/null +++ b/_posts/2020-12-10-integrating-travel-with-elixir-at-duffel.markdown @@ -0,0 +1,44 @@ +--- +layout: post +title: Integrating travel with Elixir at Duffel +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Duffel. +logo: /images/cases/logos/duffel.png +tags: api integration xml +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Duffel](https://duffel.com/) is building the new software backbone of the travel industry. Their first product is the Flights API, a developer-friendly platform that enables any business to instantly search flights, make bookings, reserve seats, and sell paid extras like checked bags. Duffel is connected to more than 20 of the world’s largest airlines, including American Airlines, Emirates, and Lufthansa. The company was founded in November 2017 and in 2019 it opened a private beta of their API and has raised $56M total in funding. It now has 40 employees across its offices in London and New York. This article discusses how Duffel has used Elixir as their technology of choice to modernize an industry built on old standards and outdated stacks. If you are interested in shaping the future of travel, [Duffel is hiring](https://duffel.com/careers). + +![Duffel](/images/cases/bg/duffel.png) + +## Why Elixir? + +Today, to access flights and prices from airlines, companies have to go through a Global Distribution System (GDS), often using decades-old systems such as Amadeus and Sabre. Steve Domin, Duffel's founder / CEO, explains: "The airline industry runs on a legacy data exchange standard called [EDIFACT standard](https://en.wikipedia.org/wiki/EDIFACT) and only recently moved to a ‘modern’ SOAP/XML world. Any integration work with a GDS or an airline is always scheduled to take months, and this creates a very high barrier to entry for new businesses." + +At its heart, Duffel is building the new operating system for travel. A single request to Duffel's API may translate into a chain of dozens of requests to different airlines. The response of each request needs to be parsed, normalized, and potentially be hydrated with more outgoing requests. All of this while managing slow responses, timeouts, large data payloads, and more. These challenges made it clear to Steve that Elixir would be a great fit: "We are building a highly concurrent platform with intensive data trafficking. From day one, it was clear the Erlang VM would be a great fit, as it was designed for telecommunication with similar requirements in mind." They chose the Erlang VM, alongside [the Phoenix web framework](https://phoenixframework.org/) and [the Ecto database library](https://github.com/elixir-ecto/ecto) as their stack to launch their initial JSON API. They leverage Elixir's standard library for most of their concurrent work and [the Saxy library for XML parsing](https://github.com/qcam/saxy). + +## Growing with Open Source + +When Steve co-founded the company in November 2017, he already had plenty of experience with Elixir. Steve started using the language before it reached 1.0, back in 2013. He started his journey by hacking on [Dynamo](https://github.com/devinus/dynamo), Phoenix's ancestor, and eventually introduced Elixir at his previous company, by using it for an internal project. He also organized meet-ups in London and contributed to Open Source projects, including some of his own, such as [Swoosh](https://github.com/swoosh/swoosh). + +The founders joined [Y Combinator](https://www.ycombinator.com/) in Summer 2018. Once they came back to London, they hired Alan Kennedy as their first engineer. Alan first heard about Elixir when he and Steve were colleagues at GoCardless. Alan kept an eye on it but never actively used it until he joined Duffel. Alan recalls struggling to jump from a language that promotes mutability to an immutable language like Elixir. Once everything clicked, he acknowledged the new programming model is conceptually much simpler. + +Since then, the company has grown with a mixture of fresh and experienced engineers, including nearly 70% of the engineering organisation programming in Elixir. + +Johanna Larsson is one of the most recent engineers to join Duffel. She had already spoken at Elixir Conferences and made meaningful contributions to the ecosystem, such as [the HexDiff project](https://diff.hex.pm/), before she was hired. In her opinion, one of Elixir's biggest assets is the community, which she considers welcoming and supportive. + +Duffel has often been able to leverage the ecosystem and reach out to existing solutions. However, they don't shy away from creating their own and open-sourcing them whenever it makes sense. Overall, the Duffel team has contributed to many areas of the ecosystem. Besides the previously mentioned Swoosh and HexDiff projects, their team members created [Hammox](https://github.com/msz/hammox), [Bigflake](https://github.com/stevedomin/bigflake), the company's own [Paginator](https://github.com/duffelhq/paginator/) library, and others. + +## Upcoming challenges + +Duffel engineers have many interesting and exciting challenges ahead of them. For example, as more developers start using the product, they will begin to hit some rate-limits imposed by airlines that they haven't yet exercised. As one would expect, different airlines have different rules and various constraints, and providing a unified solution has its hurdles. + +Some of the upcoming improvements are related to their usage of umbrella projects. Duffel started as a monolith, but they eventually migrated to Elixir's umbrella projects - a mono-repo implementation within Elixir's tooling - as soon as Phoenix v1.4 was released. Their primary motivation was to separate the communication with different airlines into different services. In the beginning, the services were clear in Steve's head, but as the team grew, they experienced friction enforcing those boundaries, which led to cyclic dependencies. + +Luckily, Elixir v1.11 started emitting warnings for cyclic and undeclared dependencies between applications, which forced the Duffel team to revisit the areas that were not strict in the past to increase the quality of the codebase in the long term. + +The team is also always exploring how to improve their APIs by bringing new approaches and technologies, such as streaming and GraphQL, as well as intelligent ways to optimize their integrations. If you are interested in tackling these and many other challenges while reshaping the travel industry, you can [learn more about Duffel's engineering opportunities](https://duffel.com/careers). diff --git a/_posts/2021-01-13-orchestrating-computer-vision-with-elixir-at-v7.markdown b/_posts/2021-01-13-orchestrating-computer-vision-with-elixir-at-v7.markdown new file mode 100644 index 000000000..8e402cae7 --- /dev/null +++ b/_posts/2021-01-13-orchestrating-computer-vision-with-elixir-at-v7.markdown @@ -0,0 +1,68 @@ +--- +layout: post +title: Orchestrating computer vision with Elixir at V7 +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at V7. +logo: /images/cases/logos/v7.png +tags: computer-vision phoenix +redirect_from: /blog/2021/01/13/orchestrating-computer-vision-with-elixir/ +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[V7](https://www.v7labs.com) is a web platform to create the sense of sight. A hub for machine learning and software engineers to develop their computer vision projects with data set management, image/video labeling, and one-click model training to automate any visual task. + +Founded in 2018 by Alberto Rizzoli and Simon Edwardsson, V7 uses Elixir, Phoenix, and Cowboy to power their web platform, responsible for managing large amounts of data and orchestrating dozens of Python nodes to carry out machine learning jobs. They have [recently closed a $3M seed round](https://www.notion.so/V7-Labs-raises-3-million-to-empower-AI-teams-with-automated-training-data-workflows-2c9b36d2043e44f3b536efae0a204632), and they are currently [hiring backend engineers to augment their Elixir team](https://www.v7labs.com/working-at-v7). + +![V7](/images/cases/bg/v7.png) + +## Visual tasks + +Throughout the years, we have been continuously automating visual tasks to speed up manual processes and reduce the rate of errors. For example: + + * Routine inspection of infrastructure: oil pipelines and offshore oil rigs require constant examination against corrosion. Once there is too much rust, it can damage the pipeline and cause leakage. Nowadays, you can use drones to take pictures and automate the detection of oxidated spots. + + * Medical examination: there is a growing use of digital pathology to assist doctors in diagnosing diseases. For example, during a biopsy of possible liver cancer, doctors use a microscope to visualize human tissue and stitch together an image of the cells, which are then individually analyzed. AI can double-check these images and help speed up problematic cells in case of positives. + + * Agriculture and farming: a wine producer may want to count grapes in a vineyard to estimate the wine production for a given season with higher precision. Farmers may use video to assess the health and the amount of exercise on free-range chickens and pigs. + + * Visual automation also plays a growing role in quality assurance and robotics: a fast-food manufacturer can use cameras to identify fries with black spots, while harvesters may use robots to pick apples from trees. + +Neural networks are at the heart of these tasks, and there is a growing need to automate the creation of the networks themselves. + +## Automating AI + +Training a neural network for image and video classification often requires multiple steps. First, you annotate images and frames with bounded-boxes, polygons, skeletons, and many other formats. The annotations are then labeled and used to train computer vision models. Labeled annotations are also used to verify models against biases, outliers, and over/underfitting. + +For many AI companies, this process exists in a loop as they continuously refine datasets and models. V7 helps teams manage and automate these steps, accelerating the creation of high-quality training data by 10-100x. Users may then export this data or use it to create neural networks directly via the platform. + + + +V7 uses Elixir to orchestrate all of these tasks. The front-end is a Vue.js application that talks to a [Phoenix-powered](https://phoenixframework.org/) API. The Phoenix application has to work with a large amount of data across a wide variety of formats. For example, a microscope outputs images in a different format, often proprietary, than a regular laboratory camera. + +To perform all the machine learning tasks, V7 has a cluster of Python nodes orchestrated by an Elixir application running the [Cowboy](https://github.com/ninenines/cowboy/) webserver. Once a Python node comes up, it establishes a WebSocket connection with Cowboy and sends how much memory, CPU, GPU, and other relevant data it has available. + +The Phoenix-powered backend communicates with the orchestrator using another Erlang VM-based technology: [RabbitMQ](https://www.rabbitmq.com/). For example, when the user tasks to auto-annotate an image, the Vue.js front-end sends a REST request to Phoenix. Phoenix then enqueues a message on RabbitMQ with the image's location, typically an Amazon S3 bucket. The orchestrator picks this message up, finds an available Python node, and delivers the relevant instructions via WebSockets. + +## Ecosystem and Infrastructure + +Other tools used by the V7 team are [Broadway](https://github.com/dashbitco/broadway) and the Erlang Distribution. + +V7 has to process and normalize images and videos. For these, they have a separate service that receives RabbitMQ messages and invokes [ImageMagick](https://imagemagick.org/) or [FFmpeg](https://ffmpeg.org/) accordingly. They use Broadway to receive RabbitMQ messages and to execute these tasks concurrently. + +The Erlang Distribution helps them broadcast information across nodes. Since they store their multimedia data on S3, they need to generate pre-signed URLs whenever the user wants to see an image or video. However, if users are routed to a different node, they would get a different URL, which would force them to download the asset again. To address this, they use the Erlang Distribution to communicate which URLs they have generated and for which purposes. + +Overall, their backend runs on Amazon ECS on about four nodes, which talk directly to PostgreSQL. The largest part of their infrastructure is the Python cluster, which takes up to two dozens of machines. + + +## Learning and Hiring + +Elixir has been present inside the company since day one, back in August 2018. Andrea Azzini, the first engineer at V7, was the one responsible for introducing it. He believed the language would be a good fit for the challenges ahead of them based on his experience running Elixir in production. + +Simon Edwardsson, their CTO, had to learn the language as they developed the system, but he was able to get up and running quickly, thanks to his previous experiences with Python and Haskell. He remarks: "As a team, we were more familiar with Django, but we were concerned it would not handle well the amount of data and annotations that we manage - which could lead to rewrites or frustrations down the road. From this perspective, the investment in Elixir was worth it, as we never had to do major changes on our backend since we started." + +Part of this is thanks to Phoenix's ability to provide high-level abstractions while making its building blocks accessible to developers: "While there is magic happening inside Phoenix, it is straight-forward to peek under the hood and make sense of everything." + +V7 has recently welcomed a new Elixir engineer to their team, making it a total of four, and they are looking for more developers interested in joining them. Historically, more engineers have applied to their machine learning positions, but they also believe many Elixir developers are prepared but don't consider themselves ready. Simon finishes with an invitation: “We are primarily looking for backend engineers with either existing Elixir experience or willingness to learn on the job. If you are interested in automating computer vision across a large range of industries, [we welcome you to get in touch](https://www.v7labs.com/working-at-v7)." diff --git a/_posts/2021-02-03-social-messaging-with-elixir-at-community.markdown b/_posts/2021-02-03-social-messaging-with-elixir-at-community.markdown new file mode 100644 index 000000000..e105822f1 --- /dev/null +++ b/_posts/2021-02-03-social-messaging-with-elixir-at-community.markdown @@ -0,0 +1,72 @@ +--- +layout: post +title: Social messaging with Elixir at Community +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Community. +logo: /images/cases/logos/community.png +tags: messaging broadway +redirect_from: /blog/2021/02/03/social-messaging-with-elixir/ +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Community](https://community.com/) is a platform that enables instant and direct communication with the people you want to reach, using the simplicity of text messaging. Used by names like Paul McCartney, Metallica, and Barack Obama, Community connects small businesses, stars, and high-profile individuals directly to their audiences. + +Community is powered by the Erlang Ecosystem, with Elixir and RabbitMQ playing central roles. This article gives an overview of the system and the tools used to handle spikes of million of users caused by events such as this tweet: + + + +## The first steps with Elixir + +Tomas Koci and Ustin Zarubin were the two engineers behind Community's initial implementation. The company was pivoting from a product they had written in Go and they felt the language was not expressive enough for the products they were building. So when faced with the challenge of developing a social messaging platform on top of SMS, they were open to trying a different stack. + +Their first encounter with Elixir was a casual one. They were chatting about the challenges ahead of them when their roommate mentioned Elixir. Shortly after, things started to click. They both had a physics background, so they found the functional paradigm quite intuitive. The Erlang VM also has its origins in telecommunications, and they were building a telecom centric product, which gave them more confidence. + +Besides the technological aspect, they also began to be active in the Elixir community. Tomas recaps: "we started attending the Elixir meetups happening here in Chattanooga. We met many developers, heard about production cases, and learned how companies like Bleacher Report were using Elixir at scale". From then on, they were sold on giving Elixir a try. + +They started their prototype in January 2018, with the intent of onboarding dozens of users. They were learning Elixir while developing the system and reaching out to potential users. + +Their first challenge was in May 2018, when one of their users announced his phone number, managed by Community, to millions of viewers. Tomas still remembers that day: "It was a Saturday night, around 11:00 pm when we saw an influx of users. It caught us by surprise and, after 10 hours, more than 400 thousand users had signed up". This influx of users stressed the system in unexpected ways, especially when it came to their upstream integrations. They had to patch the system to ensure they would not overload external systems or run over API limits they were required to conform to. + +This event also gave them insights into the types of spikes and traffic patterns the system would have to handle at scale. Early engineering hire Jeffrey Matthias urged them to break their application into different services, making it easy to scale each service individually, and he and Tomas decided to have those services communicate via message queues. + +## The next millions of users + +By October 2018, the company received funding and the newly-hired engineering team of five people, began to split the original application into services that could handle sharp increases in demand and operate at scale. Shortly after, they had their next challenge in hand: Metallica had just signed up with the platform and they were going to do an early launch with their fans on Feb 1st, 2019. + +The team is glad to report the announcement was a success with no hiccups on their end. They were then five backend engineers who tackled everything from architectural design and development to setting up and operating the whole infrastructure. + +Community was officially unveiled in May 2019, [attracting hundreds of music stars shortly after](https://www.billboard.com/amp/articles/business/8543190/why-hundreds-music-stars-giving-fans-phone-numbers-community-app). Fourteen months later, [Barack Obama tweeted to millions his phone number powered by Community](https://twitter.com/barackobama/status/1308769164190941187). + +## The current architecture + +Today, more than 60 services with distinct responsibilities power Community, such as: + +* A message hub between community leaders and members +* User data management +* Media services (video, audio, images) +* Systems for Community's internal team +* Data science and machine learning +* Billing, administration, etc + +The vast majority of those services run Elixir, with Python covering the data science and machine learning endpoints, and Go on the infrastructure side. + +[RabbitMQ](https://www.rabbitmq.com/) handles the communication between services. The Erlang-backed message queue is responsible for broadcasting messages and acting as [their RPC backbone](https://andrealeopardi.com/posts/rpc-over-rabbitmq-with-elixir/). Messages between services are encoded with Protocol Buffers via [the protobuf-elixir library](https://github.com/elixir-protobuf/protobuf). + +Initially, they used [the GenStage library](http://github.com/elixir-lang/gen_stage/) to interface with RabbitMQ, but they have migrated to the higher level [Broadway](https://github.com/dashbitco/broadway) library over the last year. Andrea Leopardi, one of their engineers, outlines their challenges: "Our system has to handle different traffic patterns when receiving and delivering data. Incoming data may arrive at any time and be prone to spikes caused by specific events powered by actions within Communities. On the other hand, we deliver SMSes in coordination with partners who impose different restrictions on volumes, rate limiting, etc." + +He continues: "both GenStage and Broadway have been essential in providing abstractions to handle these requirements. They provide back-pressure, ensure that spikes never overload the system, and guarantee we never send more messages than the amount defined by our delivery partners". As they implemented the same patterns over and over in different services, they found Broadway to provide the ideal abstraction level for them. + +Their most in-demand service, the message hub, is powered by only five machines. They use [Apache Mesos](https://mesos.apache.org/) to coordinate deployments. + +## Growing the team + +Community's engineering team has seen stable growth over the last two years. Today they are 25 backend engineers, the majority being Elixir devs, and the company extends beyond 120 employees. + +Karl Matthias, who joined early on, believes the challenges they face and the excitement for working on a new language has been positive for hiring talent. He details: "we try to hire the best production engineers we can, sometimes they know Elixir, sometimes they don't. Our team has generally seen learning Elixir as a positive and exciting experience". + +The team is also happy and confident about the stability Elixir provides. Karl adds: "Elixir supervisors have our back every time something goes wrong. They automatically reestablish connections to RabbitMQ, they handle dropped database connections, etc. The system has never gone wrong to the point our infrastructure layer had to kick-in, which has been quite refreshing." + +The Community team ended our conversation with a curious remark. They had just shut down their first implementation of the system, the one that received a sudden spike of four hundred thousand users on a Saturday night. Tomas concludes: "it is pretty amazing that the service we implemented while learning Elixir has been running and operating in production just fine, even after all of these milestones. And that's generally true for all of our services: once deployed, we can mostly forget about them". diff --git a/_posts/2021-04-02-marketing-and-sales-intelligence-with-elixir-at-pepsico.markdown b/_posts/2021-04-02-marketing-and-sales-intelligence-with-elixir-at-pepsico.markdown new file mode 100644 index 000000000..dc81c1aa8 --- /dev/null +++ b/_posts/2021-04-02-marketing-and-sales-intelligence-with-elixir-at-pepsico.markdown @@ -0,0 +1,56 @@ +--- +layout: post +title: Marketing and sales intelligence with Elixir at PepsiCo +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at PepsiCo. +logo: /images/cases/logos/pepsico.png +tags: biz-intelligence phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +PepsiCo is one of the world's leading food and beverage companies serving more than 200 countries and territories around the world. Today Elixir is used at varying capacities inside PepsiCo by six different teams. This article explores how the Search Marketing and Sales Intelligence Platform teams adopted and use Elixir to build internal tools. + +Although we will explore only two teams in this article, [PepsiCo is hiring Elixir engineers across multiple teams](https://www.pepsicojobs.com/teams-ecommerce). Let's get started. + +![PepsiCo](/images/cases/bg/pepsico.jpg) + +## The first steps + +The project that would become the first Elixir project and open the door for future Elixir applications inside PepsiCo was started by Jason Fertel back in 2016. + +Initially, the application provided workflow automation for managing search marketing operations on multiple web platforms. The product was a success and ended up integrated into PepsiCo in 2018. + +Now, the Elixir application plays a central role in a data pipeline that empowers PepsiCo's marketing and sales teams with tools to query, analyze, and integrate with several search marketing partners. + +The pipeline starts with the Data Engineering team, which collects and stores data into [Snowflake Data Cloud](https://www.snowflake.com/). The Elixir application reads data from Snowflake’s platform, pre-process, and stores it in two databases: [PostgreSQL](https://www.postgresql.org/) or [Apache Druid](https://druid.apache.org/), according to the data characteristics. Finally, a Phoenix application serves this data to internal teams and communicates directly with third-party APIs. + +## Why Elixir? + +Elixir helps PepsiCo eCommerce focus and get things done fast. “Elixir allows our team to develop quickly with confidence,” says David Antaramian, a Software Engineering Manager at PepsiCo. “In turn, that lets us deliver value to the business quickly, and it’s the reason we’ve stuck with the language. Whether it’s streaming changes to the front-end or orchestrating concurrent data operations across multiple storage systems, Elixir offers a robust developer experience that translates to a great consumer experience.” + +Different Elixir features came together to help the PepsiCo team build compelling development and user experiences. Thanks to its functional and extensible aspects, PepsiCo used Elixir to create a domain-specific language that translates business queries into data structures sent to different stores. This gave them a stable foundation where they can continually add new queries and integrations, even as they grow in complexity. + +Furthermore, the reports generated by PepsiCo's marketing and sales teams often have to query different tables or even separate storages, all while juggling long-running connections to different third-party APIs. Elixir's programming model, inherited from the Erlang Virtual Machine, makes it trivial to run all of these operations concurrently, leading to fast and rich user interactions while the development team remains focused on delivering features. + +## Libraries and frameworks + +David Antaramian is quick to praise the Erlang runtime and its standard library. He says: "Since we are working with large amounts of data, it is also essential to avoid hitting the database whenever possible. Thankfully, Erlang ships with an in-memory table storage called [ETS](http://www.erlang.org/doc/man/ets.html), which we use to store hundreds of thousands of rows". + +The Erlang standard library was also handy when communicating to some data stores. In particular, the Snowflake platform requires ODBC connections. The PepsiCo team built a library called [Snowflex](https://github.com/pepsico-ecommerce/snowflex), designed on top of [Erlang's built-in ODBC drivers](http://www.erlang.org/doc/man/odbc.html). + +The Elixir ecosystem nicely complements the Erlang one. The front-end, written in React, talks to the server via [the Absinthe GraphQL toolkit](http://absinthe-graphql.org/) running on top of [the Phoenix web framework](http://phoenixframework.org/). The [Ecto database library](https://github.com/elixir-ecto/ecto) manages the communication to PostgreSQL. They also use the [esaml](https://github.com/handnot2/esaml) and [Samly](https://github.com/handnot2/samly) libraries to provide authentication within PepsiCo's organization - another example of leveraging the tools within both Erlang and Elixir communities. + +Finally, the team also recognizes the [Erlang Ecosystem Foundation](https://erlef.org/)'s efforts, which PepsiCo are sponsors of, particularly the Observability Working Group. David remarks: "The adoption of Telemetry by the ecosystem has been a massive help in bringing monitoring visibility and metrics to our system. Now when we see spikes in one place, we can easily correlate them with other system parts". + +## Hiring + +Today there are approximately 40+ Elixir engineers within PepsiCo distributed across six teams. Eight of those engineers are part of the Search Marketing and Sales Intelligence Platform teams. + +While the team recognizes that there aren't as many Elixir engineers compared to communities like JavaScript, they were effective in hiring qualified Elixir candidates. Chase Gilliam, a Software Engineering Manager at PepsiCo, explains: "We have met many engineers that, like us, found Elixir due to being burned out by previous experiences. So when it came to hiring, many Elixir candidates had a mindset similar to ours, which ultimately sped up the process." + +This initial group of Elixir engineers paved the way for the language's growth inside PepsiCo. David adds: "At first, we looked for engineers with Elixir experience to help us build a team that could guide other developers. Then we extended the pool to anyone who has a functional programming background and now to developers with either Ruby or Erlang experience. However, if someone is the right candidate, we onboard them even if they have no Elixir experience and train them". He continues: "We also make extensive use of the learning resources available in the community, such as conferences, books, online courses, and others." + +As the team grew, they adopted best practices and saw the quality of the codebase improve at the same time. Chase concludes: "At the beginning, there were some large modules in our application. Luckily, refactoring in a functional programming language is straightforward, thanks to immutability and limited side-effects. Adopting tools like Credo, ExDoc, and the code formatter was also essential to standardize how we use Elixir internally." For those interested in learning more about the different use cases for Elixir inside PepsiCo and help continue its growth, [they are hiring](https://www.pepsicojobs.com/teams-ecommerce). diff --git a/_posts/2021-05-19-elixir-v1-12-0-released.markdown b/_posts/2021-05-19-elixir-v1-12-0-released.markdown new file mode 100644 index 000000000..676f3d2e2 --- /dev/null +++ b/_posts/2021-05-19-elixir-v1-12-0-released.markdown @@ -0,0 +1,102 @@ +--- +layout: post +title: Elixir v1.12 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.12 is out with improvements to scripting, tighter Erlang/OTP 24 integration, stepped ranges, and dozen of new functions across the standard library +--- + +Elixir v1.12 is out with improvements to scripting, tighter Erlang/OTP 24 integration, stepped ranges, and dozen of new functions across the standard library. Overall this is a small release, which continues our tradition of bringing Elixir developers quality of life improvements every 6 months. Some of these improvements directly relates with the [recent efforts of bringing Numerical Computing to Elixir](https://dashbit.co/blog/nx-numerical-elixir-is-now-publicly-available). + +Elixir v1.12 requires Erlang/OTP 22+. We also recommend running `mix local.rebar` after installation to upgrade to the latest Rebar versions, which includes support for Erlang OTP/24+. + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, `noscript` tags with the proper links will be shown. + +## Scripting improvements: `Mix.install/2` and `System.trap_signal/3` + +Elixir v1.12 brings new conveniences for those using Elixir for scripting (via `.exs` files). Elixir has been capable of managing dependencies for a quite long time, but it could only be done within Mix projects. In particular, the Elixir team is wary of global dependencies as any scripts that rely on system packages are brittle and hard to reproduce whenever your system changes. + +`Mix.install/2` is meant to be a sweet spot between single-file scripts and full-blown Mix projects. With `Mix.install/2`, you can list your dependencies at the top of your scripts. When you execute the script for the first time, Elixir will download, compile, and cache your dependencies before running your script. Future invocations of the script will simply read the compiled artifacts from the cache: + +```elixir +Mix.install([:jason]) +IO.puts(Jason.encode!(%{hello: :world})) +``` + +`Mix.install/2` also performs protocol consolidation, which gives script developers an option to execute their code in the most performant format possible. Note `Mix.install/2` is currently experimental and it may change in future releases. + +Furthermore, `Mix.install/2` pairs nicely with Livebook, a newly announced project that brings interactive and collaborative notebook projects to Elixir. With Livebook and `Mix.install/2`, you can bring dependencies into your code notebooks and ensure they are fully reproducible. [Watch the Livebook announcement to learn more](https://www.youtube.com/watch?v=RKvqc-UEe34). + +Another improvement to scripting is the ability to trap exit signals via `System.trap_signal/3`. All you need is the signal name and a callback that will be invoked when the signal triggers. For example, ExUnit leverages this functionality to print all currently running tests when you abort the test suite via SIGQUIT (`Ctrl+\\ `). You can see this in action when running tests in the Plug project below: + + + +This is particularly useful when your tests get stuck and you want to know which one is the culprit. + +**Important**: Trapping signals may have strong implications on how a system shuts down and behaves in production and therefore it is extremely discouraged for libraries to set their own traps. Instead, they should redirect users to configure them themselves. The only cases where it is acceptable for libraries to set their own traps is when using Elixir in script mode, such as in `.exs` files and via Mix tasks. + +## Tighter Erlang/OTP 24 integration + +[Erlang/OTP 24 ships with JIT compilation](https://blog.erlang.org/My-OTP-24-Highlights/) and Elixir developers don't have to do anything to reap its benefits. There are many other features in Erlang/OTP 24 to look forwards to and Elixir v1.12 provides integration with many of them: such as support for 16bit floats in bitstrings as well as performance improvements in the compiler and during code evaluation. + +Another excellent feature in Erlang/OTP 24 is the implementation of [EEP 54](http://www.erlang.org/eeps/eep-0054.html), which provides extended error information for many functions in Erlang's stdlib. Elixir v1.12 fully leverages this feature to improve reporting for errors coming from Erlang. For example, in earlier OTP versions, inserting an invalid argument into an ETS table that no longer exists would simply error with `ArgumentError`: + + + +However, in Elixir v1.12 with Erlang/OTP 24: + + + +Finally, note Rebar v2 no longer works on Erlang/OTP 24+. Mix defaults to Rebar v3 since Elixir v1.4, so no changes should be necessary by the vast majority of developers. However, if you are explicitly setting `manager: :rebar` in your dependency, you want to move to Rebar v3 by removing the `:manager` option. Compatibility with unsupported Rebar versions will be removed from Mix in the future. + +## Stepped ranges + +Elixir has had support for ranges from before its v1.0 release. Ranges support only integers and are inclusive, using the mathematic notation `a..b`. Ranges in Elixir are either increasing `1..10` or decreasing `10..1` and the direction of the range was always inferred from the first and last positions. Ranges are always lazy as its values are emitted as they are enumerated rather than being computed upfront. + +Unfortunately, due to this inference, it is not possible to have empty ranges. For example, if you want to create a list of `n` elements, you cannot express it with a range from `1..n`, as `1..0` (for `n=0`) is a decreasing range with two elements. + +Elixir v1.12 supports stepped ranges via [the `first..last//step` notation](https://hexdocs.pm/elixir/1.12/Kernel.html#..///3). For example: `1..10//2` will emit the numbers `1`, `3`, `5`, `7`, and `9`. You can consider the `//` operator as an equivalent to "range division", as it effectively divides the number of elements in the range by `step`, rounding up on inexact scenarios. Steps can be either positive (increasing ranges) or negative (decreasing ranges). Stepped ranges bring more expressive power to Elixir ranges and they elegantly solve the empty range problem, as they allow the direction of the steps to be explicitly declared instead of inferred. + +As of Elixir v1.12, implicitly decreasing ranges are soft-deprecated and warnings will be emitted in future Elixir versions based on our [deprecation policy](https://hexdocs.pm/elixir/compatibility-and-deprecations.html#deprecations). + +## `then/2` and `tap/2` + +Two new functions have been added to `Kernel` module, in order to ease working with pipelines. [`tap/2`](https://hexdocs.pm/elixir/1.12/Kernel.html#tap/2) passes the given argument to an anonymous function, returning the argument itself. [`then/2`](https://hexdocs.pm/elixir/1.12/Kernel.html#then/2) passes the given argument to an anonymous function, returning the result. The following: + +```elixir +"hello world" +|> tap(&IO.puts/1) +|> then(&Regex.scan(~r/\w+/, &1)) +``` + +Is equivalent to this: + +```elixir +"hello world" +|> (fn x -> + IO.puts(x) + x + end).() +|> (&Regex.scan(~r/\w+/, &1)).() +``` + +Both `tap/2` and `then/2` are implemented as macros, and compiler improvements available on Erlang/OTP 24 ensure the intermediate anonymous functions is optimized away, which guarantees the idioms above do not have any performance impact on your code. + +## IEx improvements + +IEx got two important quality of life improvements in this release. Hitting tab after a function invocation will show all of the arguments for said function and it is now possible to paste code with pipelines in the shell. See both features in action below: + + + +## Additional functions + +Elixir v1.12 has also added many functions across the standard library. The `Enum` module received additions such as `Enum.count_until/2`, `Enum.product/1`, `Enum.zip_with/2`, and more. The `Integer` module now includes `Integer.pow/2` and `Integer.extended_gcd/2`. + +The `Code` module got a [`cursor_context/2`](https://hexdocs.pm/elixir/1.12/Code.html#cursor_context/2) function, which is now used to power `IEx` autocompletion and it is [used by projects such as Livebook to provide intellisense](https://user-images.githubusercontent.com/17034772/115117125-533b2900-9f9d-11eb-94a9-a2cf2ccb7388.mp4). + +The EEx application has also been extended to provide metadata on text segments. This has enabled the Surface and Phoenix LiveView teams to implement [a new template language called HEEx](https://github.com/phoenixframework/phoenix_live_view/pull/1440), which validates both HTML and EEx. Finally, the `Registry` module supports the `:compressed` option, which is useful for GraphQL applications managing hundreds of thousands of subscriptions via [Absinthe](http://absinthe-graphql.org/). + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.12.0). Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2021-06-02-social-virtual-spaces-with-elixir-at-mozilla.markdown b/_posts/2021-06-02-social-virtual-spaces-with-elixir-at-mozilla.markdown new file mode 100644 index 000000000..51244a2e2 --- /dev/null +++ b/_posts/2021-06-02-social-virtual-spaces-with-elixir-at-mozilla.markdown @@ -0,0 +1,54 @@ +--- +layout: post +title: Social virtual spaces with Elixir at Mozilla +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Mozilla. +logo: /images/cases/logos/mozilla-hubs.png +tags: virtual-spaces phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[Hubs](https://hubs.mozilla.com/) is Mozilla's take on virtual social experiences. You build your own private spaces and share them with your friends, co-workers, and community. Avatars in this space can move freely in a 3D social environment and watch videos, exchange messages, and talk to other people nearby. All you need is a browser and a microphone! + +Hubs is [fully](https://github.com/mozilla/hubs) [open source](https://github.com/mozilla/reticulum) and you can host it on your infrastructure via [Hubs Cloud](https://hubs.mozilla.com/cloud). Community managers, educators, and event organizers have been using Hubs Cloud to run virtual events and online activities tailored to their specific brandings and needs. All it takes to run your own version of Hubs is one click away - which perhaps makes Hubs the most deployed Phoenix application ever! + +![Mozilla Hubs](/images/cases/bg/mozilla-hubs.jpg) + +## From VR to Elixir + +The Hubs team started at Mozilla as the Mixed Reality team about 3.5 years ago. Their main goal was to explore ways for online social interaction via avatars and mixed reality. + +They quickly focused on building their first proof of concept, where avatars could communicate, move around, and join different rooms, everything running directly in the browser. This was a significant departure from the state of the art of Virtual Reality everywhere, as the getting started experience up to this point was cumbersome and often required steep investment in the form of VR headsets. + +The initial prototype was a success and it pushed the team to build a product. However, all communication in the proof of concept was peer-to-peer, which limited the features and experiences they could provide. Therefore the Hubs team knew they needed a capable backend technology to provide fan-out communication and coordinate how all different avatars interact within the virtual spaces. John Shaughnessy, Staff Software Engineer at Mozilla, comments: "When you get a lot of people in the same space, be it virtual or in the real world, there is never only a single conversation going on. Once you get ten or twenty different people in a room, conference calls don’t work well. In Hubs, people transition between multiple simultaneous conversations simply by moving around". + +With this bold vision in hand, they assessed their whole stack. They settled on using JavaScript with [Three.js](https://threejs.org/) in the front-end and chose [the Phoenix web framework](https://phoenixframework.org/) for the backend. Greg Fodor, who was an Engineering Manager at Mozilla at the time, explains the choice: "We first listed all of the features we had to implement, from trivial things like REST endpoints, to more complex use cases, such as chat messaging and tracking where avatars are in the virtual world. Once I started to learn Phoenix, I saw all of those features were already there! The application we were building has to manage a large number of connections with real-time low latencies, something we knew the Erlang VM was an excellent fit for". + +## In production + +Hubs went live in January 2018. Almost everything in Hubs goes through Phoenix. The only exception is the WebRTC voice channels, which are handled by designated voice servers, initially implemented with [Janus](https://janus.conf.meetecho.com/) and later ported to [MediaSoup](https://mediasoup.org/). However, the Phoenix app still manages the voice servers and how connections are assigned to them. + +The deployment is orchestrated by [Habitat](https://www.chef.io/products/chef-habitat/) and running on Amazon EC2. Habitat provides packaging and orchestration. When a voice server joins the Habitat ring, the Phoenix services receive a message and start assigning voices to voice servers. Overall they run on 4 Phoenix and 4 voice servers. + +The Elixir experience in production has been quite smooth. Dominick D'Aniello, Staff Software Engineer at Mozilla, points out some areas they discussed improving: "the Phoenix application works mostly as a proxy, so we avoid decoding and reencoding the data unless we really need to. But sometimes we have to peek at the payloads and JSON is not the most efficient format to do so." They have also considered relying more on Elixir processes and the Erlang distribution. Dominick continues: "when a new client joins, it needs to ask all other clients what is their state in the world, what they own, and what they care about. One option is to use Elixir processes in a cluster to hold the state of the different entities and objects in the virtual world". + +## Beyond Hubs + +With many large companies investing in online communication, the Mozilla team saw the possibility of virtual spaces becoming walled-gardens inside existing social platforms. This led the Hubs team to work on Hubs Cloud, with the mission to commoditize virtual spaces by allowing anyone to run their own version of Hubs with a single click. + +Hubs Cloud launched in February 2020 and it has been a hit. [New York University did its graduation ceremony on a Hubs Cloud instance](https://twitter.com/nyuniversity/status/1258401916096315399). [The IEEE Virtual Reality Conference embraced Hubs](https://www.computer.org/conferences/organize-a-conference/organizer-resources/hosting-a-virtual-event/success-stories/IEEE-VR-2020) for a more accessible and sustainable event with talks and poster sessions all happening in virtual rooms, while [the Minnesota Twins baseball team launched a Virtual Hall of Fame](https://www.twincities.com/2021/02/09/twins-set-to-launch-new-virtual-fan-experience/) on top of the platform. + +Their cloud version uses Amazon CloudFormation to instantiate Hubs inside the user's account. This approach brought different challenges to the Hubs team: "we want Hubs Cloud to be as affordable and straightforward as possible. The Phoenix app has already been a massive help on this front. We have also moved some features to Amazon Lambda and made them optional, such as image resizing and video conversion" - details John. + +Since Hubs is also open source, developers can run their own Hubs instance in whatever platform they choose or change it however they want. That's the path Greg Fodor recently took when he announced [Jel](https://jel.app/): "Jel is the video game for work. It is a mashup of Minecraft and Discord, where everything is 3D. My goal is to spark new directions and ideas to get people excited about VR". + +## Summing up + +Today, the Hubs team has 10 contributors, half of whom are developers. Their engineering team is quite general and learning Elixir happens organically: "you are motivated by the feature you are working on. If it requires changing the backend, you learn Elixir with the help of the team and then make your contribution". + +Overall, the bet on Phoenix was a successful one. Greg Fodor highlights: "The most significant benefit of Phoenix is in using a stack that excels at solving a large variety of problems. Once onboarded to Phoenix, there is a huge surface area our engineers can touch. Any feature they come up with, they can run with it. And because Hubs is open source, our contributors will also have the same experience. Overall, Elixir and Phoenix reduce the effort needed to cause the largest impact possible across our whole product". + +Lately, they have leaned even further into the ecosystem, as they have started exposing Hubs APIs over GraphQL with the help of Absinthe. They have also migrated to Phoenix v1.5 and are using the [Phoenix LiveDashboard](https://github.com/phoenixframework/phoenix_live_dashboard) to provide metrics and instrumentation to Hubs Cloud users. diff --git a/_posts/2021-07-29-bootstraping-a-multiplayer-server-with-elixir-at-x-plane.markdown b/_posts/2021-07-29-bootstraping-a-multiplayer-server-with-elixir-at-x-plane.markdown new file mode 100644 index 000000000..e0563549c --- /dev/null +++ b/_posts/2021-07-29-bootstraping-a-multiplayer-server-with-elixir-at-x-plane.markdown @@ -0,0 +1,56 @@ +--- +layout: post +title: Bootstrapping a multiplayer server with Elixir at X-Plane +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at X-Plane. +logo: /images/cases/logos/x-plane.png +tags: multiplayer udp otp +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +[X-Plane 11](https://www.x-plane.com/) is the world's most comprehensive and powerful flight simulator for personal computers and mobile devices. X-Plane is not a game but an engineering tool created by Laminar Research that can be used to predict the flying qualities of fixed- and rotary-wing aircraft with incredible accuracy. The X-Plane franchise comes in both consumer and FAA-certifiable professional versions. + +Recently, the X-Plane team took on the challenge of adding a multiplayer experience with the goal of hosting north of 10000 users in the same session. This article explores why they chose Elixir and how a team of one developer - without prior language experience - learned the language and deployed a well-received multiplayer experience in 6 months. The overall solution features a brand new open-source implementation of the RakNet communication protocol in Elixir and overperforms the original requirements when put under unexpected load. + +![X-Plane](/images/cases/bg/x-plane.jpg) + +## Requirements + +The X-Plane team has offered peer-to-peer multiplayer in the simulator for a long time but never server-hosted multiplayer. This was a new journey for them and they had complete freedom to pick the technology stack. [According to their blog post](https://developer.x-plane.com/2021/01/have-you-heard-the-good-news-about-elixir/), their goals were: + +1. To build a rock-solid server with error isolation. For example, an exception during a client update should not bring the whole server down. + +2. To implement a single shared world that can scale to tens of thousands of concurrent pilots. + +3. To iterate quickly: because this was the first time the Laminar Research team offered a hosted multiplayer environment, they wanted to move quickly to ship this system. This would allow users to begin flying in groups immediately and serve as a platform to gauge interest in further feature development. + +4. To be fast and consistent. Multiplayer has a "soft real-time" constraint, and they need to service _all_ clients consistently and on time. Quantitatively, this means the 99th percentile response times matter a lot more than the mean or median. + +From those requirements, the need for stability and fast iteration ruled out low-level languages, even the ones in which they had plenty of in-house experience. + +The need for speed and vertical scalability excluded many modern web languages, such as Ruby and Python, where the model for scaling up is generally throwing more servers at it. It was essential to avoid synchronizing state across multiple machines, which requires more development time and worsens the user experience due to the increased latency. + +They eventually settled on three top contenders: Rust, Go, and Elixir. Elixir took the edge thanks to two exclusive features: fault tolerance and predictable latency. Both are built into the very core of the Erlang Virtual Machine - the robust platform that Elixir runs on. Tyler Young, X-Plane's engineer leading this implementation, highlights: "We wanted a stack that could max server capacity. We would rather run a 64-core machine than dozens of 4-core VMs. Saša Jurić's talk, [the Soul of Erlang and Elixir](https://www.youtube.com/watch?v=JvBT4XBdoUE), showed us that the concurrency model, process isolation, and partial restarts provided by the platform were the abstractions we were looking for." + +## Modeling multiplayer with Elixir + +Ready to give Elixir a try, Tyler picked up a couple books but soon realized the language's [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) provided the background he needed. He explains: "while the introductory guide covers the language constructs, the advanced guide on the website has you build an actual project with TCP connections, with the basic architectural patterns we would use in production." + +However, instead of jumping headfirst into the multiplayer server, he decided to give Elixir a try on a smaller problem. He wrote a web proxy to the National Oceanic and Atmospheric Administration (NOAA) weather services and put it in production. This experience taught him the importance of leveraging all of the instrumentation and metrics provided by the Erlang VM. They chose [AppSignal](https://www.appsignal.com/) to help consume and digest this information. + +Two weeks later, he started working on the server by implementing [the UDP-centric RakNet protocol in Elixir](https://en.wikipedia.org/wiki/RakNet). Unfortunately, there is little documentation, so they had to refer to the reference implementation in C++ most of the time. Luckily, thanks to its roots in telecommunication and network services, [Elixir and Erlang have built-in support for parsing binary packets](https://hexdocs.pm/elixir/Kernel.SpecialForms.html#%3C%3C%3E%3E/1), which made the task a joy. The team also mapped each UDP connection to distinct lightweight threads of execution in Elixir, which we call _processes_. Elixir processes are cheap, isolated, concurrent, and are fairly scheduled by the runtime. This design allowed the X-Plane team to fully leverage the properties of robustness and predictable latency that first attracted them to the platform. Their implementation is written on top of Erlang's [gen_udp](http://www.erlang.org/doc/man/gen_udp.html) and [is open source](https://github.com/X-Plane/elixir-raknet). + +Five months after choosing Elixir, they began welcoming beta testers into the server. The community's reaction was overwhelmingly positive, and the new multiplayer experience led to a strong uptick in the number of subscriptions as it went live a month later. + +## Deployment and keeping it simple + +At the moment, X-Plane's player base in North America is powered by a single server, running on 1 eight-core machine with 16GB of memory, although only 200MB or 300MB of memory is effectively used. Each connected player sends 10 updates a second. + +For deployments, they use a blue-green strategy, alternating between two servers of the same capacity. Tyler explains: "We are aware the Erlang VM provides hot code swapping and distribution, but we are taking the simplest route whenever possible. It is much easier for us to alternate between two servers during deployments, as the servers are stable and we don't deploy frequently. Similarly, when it comes to distribution, we prefer to scale vertically or set up different servers in different regions for players across the globe." + +Paul McCarty, who joined the project after launch, can attest to its simplicity: "even without prior Elixir experience, I was able to jump in and add new functionality to our HTTP APIs early on." Those APIs are built on top of [Plug](http://github.com/elixir-lang/plug) to power their chat services, provide information about connected users, and more. He concludes: "When adding new features, the server development is never the bottleneck." + +Paul and Tyler finished our conversation with a curious anecdote: a couple months ago, they distributed an updated client version with debug code in it. This additional code caused each connected user to constantly ping the server every 100ms, even if not in multiplayer mode. This caused their traffic to increase 1000x! They only discovered this increase 2 weeks later when they saw the CPU usage in their Elixir server went from 5% to 21%. Once they found out the root cause and how the system handled it, they realized they didn't have to rush a client update to remove the debug code and they chose to maintain their regular release cycle. At the end of the day, it was a perfect example of the confidence they gained and built around the language and platform. diff --git a/_posts/2021-11-10-embracing-open-data-with-elixir-at-the-ministry-of-ecological-transition-in-france.markdown b/_posts/2021-11-10-embracing-open-data-with-elixir-at-the-ministry-of-ecological-transition-in-france.markdown new file mode 100644 index 000000000..7326c7d57 --- /dev/null +++ b/_posts/2021-11-10-embracing-open-data-with-elixir-at-the-ministry-of-ecological-transition-in-france.markdown @@ -0,0 +1,70 @@ +--- +layout: post +title: Embracing open data with Elixir at the Ministry of Ecological Transition in France +authors: +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at the Ministry of Ecological Transition in France. +logo: /images/cases/logos/met-france.svg +tags: open-data gov phoenix +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all cases](/cases.html) we have published so far.* + +A group of initiatives towards innovation and open data has given the opportunity for Elixir to play a central role in exploring, validating, and visualizing transportation data across all of France. This article will show how Elixir came to power [the National Access Point for transport data in France](https://transport.data.gouv.fr/) and explore why it continues to be an excellent fit thanks to its real-time libraries, educational tooling, and orchestration capabilities. + +![Sample map of transports](/images/cases/bg/met-france.png) + +## State Startups + +In 2013, the French Government launched a Digital Services incubator, called [beta.gouv.fr](https://beta.gouv.fr), to spread the culture of digital innovation throughout the administration. They do this through State Startups. + +State Startups are a match between a team and a mission. They help "intrapreneurs" - public servants who identify frictions and opportunities to improve the lives of their fellow citizens - tackle real-world challenges alongside a team of experts. This team of 2 to 4 people has six months to build a proof of concept ([see official website](https://beta.gouv.fr/en/)). + +The insight is: instead of trying to convince other stakeholders to work towards specific goals, it is better to empower innovation and those who want to drive change. Those individuals are given a budget and the autonomy to choose the technology and assemble their team. In exchange, they must open-source and publicly document all code, costs, metrics, and statistics. + +The first State Startup was [data.gouv.fr](https://www.data.gouv.fr/en), which transformed France's open data efforts from a catalog of spreadsheets into a social platform that closed the gap between the citizens consuming the data and the institutions providing them. The tool is fully [open-source](https://github.com/opendatateam/udata), allowing other countries to use it in production too. + +At the time of writing, [261 State Startups](https://beta.gouv.fr/startups/) have been launched and are in various states of development. + +## Elixir drives by + +In 2017, a team was assembled to begin a new State Startup focused on transportation data. An incoming European delegated regulation would make it mandatory for institutions and corporations to make transportation data public. The State Startup aimed at preparing the ecosystem actors for this regulatory change. + +To address this, the team decided to build a web application to search and visualize the existing transportation data available in [data.gouv.fr](https://www.data.gouv.fr). They initially targeted public transportation information provided by cities about buses, subways, and trams, all available in a static format called General Transit Feed Specification ([GTFS](https://gtfs.org/)) ([live example](https://transport.data.gouv.fr/resources/50471#visualization)). + +The two developers of the team, Vincent Lara and Mauko Quiroga, had heard about Elixir and were interested in learning more. They understood it could provide a robust but flexible and fun platform to explore the problem space. So [they bootstrapped the application](https://github.com/etalab/transport-site/commit/837a048c37ac31151b51ac09432dbcbff3917de5) with the [Phoenix web framework](https://phoenixframework.org/). + +As they developed the system, they spotted gaps and errors in the data available. So they began validating the data and reaching out to the institutions publishing them, providing direct feedback and value to the open data platform. The incubator saw the benefits from their contributions and, after a few months, they had a successful State Startup in their hands alongside the approval to continue working on their mission. + +Between 2017 and 2021, the multi-disciplinary team (one public servant, “business developers”, and technical staff) worked to increase the coverage of published transportation data and helped cities and operators to reach their technical and regulatory goals. + +## Current challenges + +In 2021, the State Startup has "graduated" from its “beta gouv” incubator and is now part of France's Ministry of Ecological Transition. Now composed by Francis Chabouis, Thibaut Barrère, and Antoine Augusti, the technical part of the team is tackling new use cases and challenges as the platform grows in terms of needs and versatility. + +Many of those are driven by the adoption of new data formats by governments and corporations. For example, [GTFS](https://github.com/google/transit/tree/master/gtfs/spec/en) provides a static (theoretical) itinerary: if a bus is currently delayed, this information would not be available in the feed. Enter the [GTFS-RT](https://github.com/google/transit/tree/master/gtfs-realtime/spec/en) format, where RT stands for real-time, to address those gaps. The General Bikeshare Feed Specification ([GBFS](https://nabsa.net/resources/gbfs/)) ([live example](https://transport.data.gouv.fr/datasets/velos-libre-service-creteil-cristolib-disponibilite-en-temps-reel/)) tracks bicycles, scooters, carpooling, etc. Plus the [SIRI](https://en.wikipedia.org/wiki/Service_Interface_for_Real_Time_Information) (Service Interface for Real-time Information) and [NeTEx](https://en.wikipedia.org/wiki/NeTEx) families of protocols. + +Some of those formats have supporting technologies (validators, converters) written in other languages (Java, Rust, etc), which would be beneficial to integrate with. The team then realized the way forward is to adapt their Elixir system to orchestrate and coordinate those subsystems. Luckily, Elixir has shown to be exceptionally well suited to this task, thanks to the underlying Erlang Virtual Machine, designed for communication systems. Francis Chabouis expands: "We currently need to integrate with internal and external services in a variety of formats. Some are static, some require periodic pulls, and others keep open connections to push data. Elixir allows us to experiment, iterate, and scale those ideas quickly". + +Overall, the data catalog now includes: + +* Timetables for buses, subways, and trains, including trips and operators, as real-time updates +* Bicycle lanes and carpooling areas +* Charging and refueling stations +* Private parking areas +* Location of rental cars, scooters, bicycles, and others + +Many of those formats also bring real-time concerns as they evolve the application to sample and show events as they happen. This is where the team is currently working at leveraging [Phoenix LiveView](http://github.com/phoenixframework/phoenix_live_view) to build the interactivity and functionality they need while keeping their stack simple and productive. + +The technical team has also recently grown to three developers, totaling seven members altogether. To prepare themselves for the new team members, Thibaut Barrère was responsible for upgrading their dependencies, including Elixir and Erlang, which were largely unchanged from 2017. While performing such changes can often be daunting in other stacks, Thibaut shares a very positive experience: "we did not see any breaking changes after performing long-due updates. Overall, the language and libraries seem quite stable and careful to avoid breaking changes. This experience gives us the confidence we can continue focusing on the needs of our users as we move forward". + +## Open data and education + +As with any other team, some challenges go beyond the technical aspects. For example, they sometimes spot companies and cities that do not make their transportation data available, which is against European Law. The team often heard concerns about making parts of their systems public, which could lead to failures in critical infrastructure. + +To address this, the team built [a small Phoenix application](https://github.com/etalab/transport-site/tree/master/apps/unlock) that works as a simple proxy to those systems. The proxy caches the data for specific intervals, helping those entities address the security and traffic concerns around their critical systems. The application uses [Cachex](https://github.com/whitfin/cachex) and provides a real-time dashboard, built with LiveView, where they can configure the system, track application load, and see cache usage data. + +Another area the team is actively investigating is how to make the data itself more accessible to developers who want to consume it. A non-trivial amount of work is required between fetching the city data, parsing it, and displaying it on a map, which can discourage developers from beginning their open data journey. To this end, they plan to assemble a collection of [Livebooks](http://github.com/livebook-dev/livebook), a recently released tool for writing code notebooks in Elixir that allows developers to get up and running quickly and obtain immediate feedback on their code. + +Thibaut remarks how the growth of the language and its ecosystem supports their application and needs: "every time we faced a new challenge, a solution was readily available to us. When we needed to orchestrate multiple subsystems, the stack excelled at it. When we required real-time features, Phoenix and LiveView had first-class support for it. Now we need to promote education and access to open data, and Livebook is shaping to be precisely what we need". diff --git a/_posts/2021-12-03-elixir-v1-13-0-released.markdown b/_posts/2021-12-03-elixir-v1-13-0-released.markdown new file mode 100644 index 000000000..486d27699 --- /dev/null +++ b/_posts/2021-12-03-elixir-v1-13-0-released.markdown @@ -0,0 +1,153 @@ +--- +layout: post +title: Elixir v1.13 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.13 is out with a focus on developer tooling +--- + +Elixir v1.13 has just been released! + +Generally speaking, new Elixir versions include improvements to its primary API, the one Elixir developers use every day, and also to the foundation that powers its tooling. In this release, however, it coincided that most new functionality centers around Elixir tooling. The result is a series of quality of life improvements that will impact Elixir developers immediately as well as in the long term. + +Let's check them out! + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, `noscript` tags with the proper links will be shown. + +## Semantic recompilation + +The feature that will most and immediately benefit all Elixir developers is the series of improvements we have made to how the compiler tracks file contents. + +Generally speaking, once a file changes, it may lead to other files in your codebase to be recompiled. In previous versions, however, Elixir made no effort to understand which parts of the file changed. This meant the smallest of changes to certain files, such as configuration files, could trigger a full project recompilation. + +This release comes with a series of improvements that better understand how your files change. In particular: + + * An Elixir file is no longer considered as changed if its size and its digest stay the same. This avoids recompiling many files when switching or rebasing branches. + + * Changing your `mix.exs` will no longer trigger a full project recompilation, unless you specifically change the configurations used by the Elixir compiler (`:elixirc_paths` and `:elixirc_options`). + + * Changing compile-time configuration files (`config/config.exs` and any other files imported from it) now only recompiles the project files that depend on the reconfigured applications, instead of a full project recompilation. However, if you change the configuration of your application itself, the whole project is still recompiled. + + * Adding, updating or removing a dependency now only recompiles the project files that depend on the modified dependency. + + * If your project has both Erlang and Elixir files, changing an Erlang file will now recompile only the Elixir files that depend on it. + +In a nutshell, Elixir went from triggering full recompilations whenever any of `mix.exs`, `config/config.exs`, `src/*`, and `mix.lock` changed on disk to semantic recompilations. Now it only fully recompiles when: + + * you change the compilation options in `mix.exs` + * you change the configuration for the current project in `config/config.exs` + +To give a more practical example, take a regular [Phoenix project](https://phoenixframework.org/). It is most likely divided in two main directories: `my_app` and `my_app_web`. Most of your usage of Phoenix' APIs will happen within the files in the `my_app_web` directory. However, if you bumped your Phoenix version or changed its configuration in previous Elixir versions, it would cause all files, in both directories, to be recompiled. With these changes, the recompilation should affect mostly the files in `my_app_web`. + +> To further clarify, the Elixir compiler is not tracking directories. It is just a consequence of how Phoenix projects are organized that most dependencies to Phoenix are within `my_app_web`. + +## Code fragments + +The [`Code`](https://hexdocs.pm/elixir/Code.html) module got a companion module called [`Code.Fragment`](https://hexdocs.pm/elixir/Code.Fragment.html). + +The `Code` module works with complete code. For example, its functions will consider the snippet `123 +` as invalid, since the right-hand side of `+` is missing. However, our tooling, such as editors, REPLs, and code notebooks must still parse and understand such snippets, in order to provide code completion, argument suggestion, etc. + +That's the goal of the `Code.Fragment` module. It contains different heuristics to analyze and return context informational of code fragments, which are code snippets that may be incomplete. + +To better show the benefits of said improvements, let's talk about IEx, Elixir's interactive shell. IEx has been rewritten to use `Code.Fragment` and, in the process, it gained new functionality as part of its autocompletion system (available by hitting TAB). For example, it can now autocomplete sigils, used to [create regexes](https://hexdocs.pm/elixir/Kernel.html#sigil_r/2) or [lists of words](https://hexdocs.pm/elixir/Kernel.html#sigil_w/2), and their terminators: + + + +Similarly, you can now autocomplete struct names and their fields: + + + +Overall, we hope the `Code.Fragment` module will become the shared foundation to power many of the tools in the ecosystem. We have also added new reflection APIs to [`Module`](https://hexdocs.pm/elixir/Module.html), which can then be used to power code intelligence features. + +## mix xref + +[`mix xref`](https://hexdocs.pm/mix/Mix.Tasks.Xref.html) is a tool that analyzes relationships between files. By analyzing the compile-time and runtime dependencies between them, it allows developers to understand what has to be recompiled whenever a file changes. + +Elixir v1.13 comes with many improvements to `mix xref`, such as: + + * `mix xref graph` now supports `--label` to be set to "compile-connected", which returns all compile-time dependencies that lead to additional transitive dependencies. + + * A new `mix xref trace FILE` subcommand receives a file and returns all dependencies in said file, including the line and what caused said dependency (a function/macro call, an alias, a struct, etc). + + * All `mix xref` subcommands support the `--fail-above` flag, which allows you to enforce your project has at most a certain number of compile-time cycles, transitive compile-time dependencies, etc. This can be useful on Continuous Integration (CI) servers. + + * `mix xref graph` now supports multiple `--sink` and `--source` to be given. + +If you haven't used `mix xref` before, it may be hard to visualize what those changes mean. If you want to learn more, you can [watch the relevant section of my ElixirConf 2021 keynote](https://youtu.be/ydjx2kKHzrM?t=772) that includes a short introduction to `mix xref`. + +Those improvements came from direct feedback from the community. A special shout out to Marc-André Lafortune for the contributions and testing. + +## Extended code formatting + +Thanks to its sigils, Elixir provides the ability of embedding snippets in other languages inside its source code. One could use it to embed XML: + + ~X""" + + + """ + +Or even [Zig](https://ziglang.org/), [via the Zigler project](https://github.com/ityonemo/zigler): + + ~Z""" + /// nif: example_fun/2 + fn example_fun(value1: f64, value2: f64) bool { + return value1 > value2; + } + """ + +However, while you can format Elixir source code with [`mix format`](https://hexdocs.pm/mix/Mix.Tasks.Format.html), you could not format the code inside snippets. + +Elixir v1.13 solves this by adding plugins to `mix format`. Plugins can teach the formatter how to format new files and how to format sigils, via the `Mix.Tasks.Format` behaviour. + +For example, imagine that your project uses Markdown in two distinct ways: via a custom `~M` sigil and via files with the `.md` and `.markdown` extensions. A custom plugin would look like this: + +```elixir +defmodule MixMarkdownFormatter do + @behaviour Mix.Tasks.Format + + def features(_opts) do + [sigils: [:M], extensions: [".md", ".markdown"]] + end + + def format(contents, opts) do + # logic that formats markdown + end +end +``` + +Now any application can use your formatter as follows: + +```elixir +# .formatter.exs +[ + # Define the desired plugins + plugins: [MixMarkdownFormatter], + # Remember to update the inputs list to include the new extensions + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}", "posts/*.{md,markdown}"] +] +``` + +We are looking forward to see how this new functionality will be used by community, especially projects like [Surface](https://github.com/surface-ui/surface) and [Phoenix LiveView](https://github.com/phoenixframework/phoenix_live_view), which provide a templating language on top of the HTML markup. + +## Other bits and bytes + +`SyntaxError` and `TokenMissingError` were improved to show a code snippet whenever possible: + + $ elixir -e "hello + * world" + ** (SyntaxError) nofile:1:9: syntax error before: '*' + | + 1 | hello + * world + | ^ + +The `Code` module has also been augmented with two functions: [`Code.string_to_quoted_with_comments/2`](https://hexdocs.pm/elixir/Code.html#string_to_quoted_with_comments/2) and [`Code.quoted_to_algebra/2`](https://hexdocs.pm/elixir/Code.html#quoted_to_algebra/2). Those functions allow someone to retrieve the Elixir AST with their original source code comments, and then convert this AST to formatted code. In other words, those functions provide a wrapper around the Elixir Code Formatter, supporting developers who wish to create tools that directly manipulate and custom format Elixir source code. + +`elixir --short-version` has been added to quickly get the Elixir version, without booting the Erlang VM. The `Task` module includes performance optimizations and [new](https://hexdocs.pm/elixir/Task.html#ignore/1) [functions](https://hexdocs.pm/elixir/Task.html#completed/1). Finally, `mix test --profile-require=time` has been added to debug loading times of test suites and the recently added [`Mix.install/2`](https://hexdocs.pm/mix/Mix.html#install#2) has been improved with new options and environment variables. + +## Learn more + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.13.0). You can also [watch my ElixirConf 2021 keynote about Elixir v1.13](https://youtu.be/ydjx2kKHzrM) to learn more. + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Have fun! diff --git a/_posts/2022-09-01-elixir-v1-14-0-released.markdown b/_posts/2022-09-01-elixir-v1-14-0-released.markdown new file mode 100644 index 000000000..7ae52d259 --- /dev/null +++ b/_posts/2022-09-01-elixir-v1-14-0-released.markdown @@ -0,0 +1,192 @@ +--- +layout: post +title: Elixir v1.14 released +authors: +- Andrea Leopardi +category: Releases +excerpt: Elixir v1.14 is out with a focus on debugging and developer experience +--- + +Elixir v1.14 has just been released. 🎉 + +Let's check out new features in this release. Like many of the past Elixir releases, this one has a strong focus on developer experience and developer happiness, through improvements to debugging, new debugging tools, and improvements to term inspection. Let's take a quick tour. + +Note: this announcement contains [asciinema](https://asciinema.org) snippets. You may need to enable 3rd-party JavaScript on this site in order to see them. If JavaScript is disabled, `noscript` tags with the proper links will be shown. + +## `dbg` + +[`Kernel.dbg/2`](https://hexdocs.pm/elixir/Kernel.html#dbg/2) is a new macro that's somewhat similar to [`IO.inspect/2`](https://hexdocs.pm/elixir/IO.html#inspect/2), but specifically tailored for **debugging**. + +When called, it prints the value of whatever you pass to it, plus the debugged code itself as well as its location. + + + +`dbg/2` can do more. It's a macro, so it *understands Elixir code*. You can see that when you pass a series of `|>` pipes to it. `dbg/2` will print the value for every step of the pipeline. + + + +## IEx + dbg + +Interactive Elixir (IEx) is Elixir's shell (also known as REPL). In v1.14, we have improved IEx breakpoints to also allow line-by-line stepping: + + + +We have also gone one step further and integrated this new functionality with `dbg/2`. + +`dbg/2` supports **configurable backends**. IEx automatically replaces the default backend by one that halts the code execution with IEx: + + + +We call this process "prying", as you get access to variables and imports, but without the ability to change how the code actually executes. + +This also works with pipelines: if you pass a series of `|>` pipe calls to `dbg` (or pipe into it at the end, like `|> dbg()`), you'll be able to step through every line in the pipeline. + + + +You can keep the default behavior by passing the `--no-pry` option to IEx. + +## dbg in Livebook + +[Livebook](https://livebook.dev/) brings the power of computation notebooks to Elixir. + +As an another example of the power behind `dbg`, the Livebook team has implemented a visual representation for `dbg` as a backend, where it prints each step of the pipeline as its distinct UI element. You can select an element to see its output or even re-order and disable sections of the pipeline on the fly: + + + +## PartitionSupervisor + +[`PartitionSupervisor`](https://hexdocs.pm/elixir/PartitionSupervisor.html) implements a new supervisor type. It is designed to help when you have a single supervised process that becomes a bottleneck. If that process' state can be easily partitioned, then you can use `PartitionSupervisor` to supervise multiple isolated copies of that process running concurrently, each assigned its own partition. + +For example, imagine you have a `ErrorReporter` process that you use to report errors to a monitoring service. + +```elixir +# Application supervisor: +children = [ + # ..., + ErrorReporter +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +As the concurrency of your application goes up, the `ErrorReporter` process might receive requests from many other processes and eventually become a bottleneck. In a case like this, it could help to spin up multiple copies of the `ErrorReporter` process under a `PartitionSupervisor`. + +```elixir +# Application supervisor +children = [ + {PartitionSupervisor, child_spec: ErrorReporter, name: Reporters} +] +``` + +The `PartitionSupervisor` will spin up a number of processes equal to `System.schedulers_online()` by default (most often one per core). Now, when routing requests to `ErrorReporter` processes we can use a `:via` tuple and route the requests through the partition supervisor. + +```elixir +partitioning_key = self() +ErrorReporter.report({:via, PartitionSupervisor, {Reporters, partitioning_key}}, error) +``` + +Using `self()` as the partitioning key here means that the same process will always report errors to the same `ErrorReporter` process, ensuring a form of back-pressure. You can use any term as the partitioning key. + +### A common example + +A common and practical example of a good use case for `PartitionSupervisor` is partitioning something like a `DynamicSupervisor`. When starting many processes under it, a dynamic supervisor can be a bottleneck, especially if said processes take a long time to initialize. Instead of starting a single `DynamicSupervisor`, you can start multiple: + +```elixir +children = [ + {PartitionSupervisor, child_spec: DynamicSupervisor, name: MyApp.DynamicSupervisors} +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +Now you start processes on the dynamic supervisor for the right partition. For instance, you can partition by PID, like in the previous example: + +```elixir +DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {MyApp.DynamicSupervisors, self()}}, + my_child_specification +) +``` + +## Improved errors on binaries and evaluation + +Erlang/OTP 25 improved errors on binary construction and evaluation. These improvements apply to Elixir as well. Before v1.14, errors when constructing binaries would often be hard-to-debug, generic "argument errors". Erlang/OTP 25 and Elixir v1.14 provide more detail for easier debugging. This work is part of [EEP 54](https://www.erlang.org/eeps/eep-0054). + +Before: + +```elixir +int = 1 +bin = "foo" +int <> bin +#=> ** (ArgumentError) argument error +``` + +Now: + +```elixir +int = 1 +bin = "foo" +int <> bin +#=> ** (ArgumentError) construction of binary failed: +#=> segment 1 of type 'binary': +#=> expected a binary but got: 1 +``` + +Code evaluation (in IEx and Livebook) has also been improved to provide better error reports and stacktraces. + +## Slicing with Steps + +Elixir v1.12 introduced **stepped ranges**, which are ranges where you can specify the "step": + +```elixir +Enum.to_list(1..10//3) +#=> [1, 4, 7, 10] +``` + +Stepped ranges are particularly useful for numerical operations involving vectors and matrices (see [Nx](https://github.com/elixir-nx/nx), for example). However, the Elixir standard library was not making use of stepped ranges in its APIs. Elixir v1.14 starts to take advantage of steps with support for stepped ranges in a couple of functions. One of them is [`Enum.slice/2`](https://hexdocs.pm/elixir/Enum.html#slice/2): + +```elixir +letters = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] +Enum.slice(letters, 0..5//2) +#=> ["a", "c", "e"] +``` + +[`binary_slice/2`](https://hexdocs.pm/elixir/Kernel.html#binary_slice/2) (and [`binary_slice/3`](https://hexdocs.pm/elixir/Kernel.html#binary_slice/3) for completeness) has been added to the `Kernel` module, that works with bytes and also support stepped ranges: + +```elixir +binary_slice("Elixir", 1..5//2) +#=> "lxr" +``` + +## Expression-based Inspection and `Inspect` Improvements + +In Elixir, it's conventional to implement the `Inspect` protocol for opaque structs so that they're inspected with a special notation, resembling this: + +```elixir +MapSet.new([:apple, :banana]) +#MapSet<[:apple, :banana]> +``` + +This is generally done when the struct content or part of it is private and the `%name{...}` representation would reveal fields that are not part of the public API. + +The downside of the `#name<...>` convention is that *the inspected output is not valid Elixir code*. For example, you cannot copy the inspected output and paste it into an IEx session. + +Elixir v1.14 changes the convention for some of the standard-library structs. The `Inspect` implementation for those structs now returns a string with a valid Elixir expression that recreates the struct when evaluated. In the `MapSet` example above, this is what we have now: + +```elixir +fruits = MapSet.new([:apple, :banana]) +MapSet.put(fruits, :pear) +#=> MapSet.new([:apple, :banana, :pear]) +``` + +The `MapSet.new/1` expression evaluates to exactly the struct that we're inspecting. This allows us to hide the internals of `MapSet`, while keeping it as valid Elixir code. This expression-based inspection has been implemented for `Version.Requirement`, `MapSet`, and `Date.Range`. + +Finally, we have improved the `Inspect` protocol for structs so that fields are inspected in the order they are declared in `defstruct`. The option `:optional` has also been added when deriving the `Inspect` protocol, giving developers more control over the struct representation. See [the updated documentation for `Inspect`](https://hexdocs.pm/elixir/Inspect.html) for a general rundown on the approaches and options available. + +## Learn more + +For a complete list of all changes, see the [full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.14.0). + +Check [the Install section](/install.html) to get Elixir installed and read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more. + +Happy debugging! diff --git a/_posts/2022-10-05-my-future-with-elixir-set-theoretic-types.markdown b/_posts/2022-10-05-my-future-with-elixir-set-theoretic-types.markdown new file mode 100644 index 000000000..11f8d06c6 --- /dev/null +++ b/_posts/2022-10-05-my-future-with-elixir-set-theoretic-types.markdown @@ -0,0 +1,140 @@ +--- +layout: post +title: "My Future with Elixir: set-theoretic types" +authors: +- José Valim +category: Announcements +excerpt: We announce and explore the possibilities for bringing set-theoretic types into Elixir. +--- + +*This is article contains excerpts from my keynotes at [ElixirConf Europe 2022](https://www.youtube.com/watch?v=Jf5Hsa1KOc8) and [ElixirConf US 2022](https://www.youtube.com/watch?v=KmLw58qEtuM).* + +In May 2022, we have celebrated 10 years since Elixir v0.5, the first public release of Elixir, was announced. + +At such occasions, it may be tempting to try to predict how Elixir will look in 10 years from now. However, I believe that would be a futile effort, because, 10 years ago, I would never have guessed Elixir would have gone [beyond excelling at web development](https://phoenixframework.org/), but also into domains such as [embedded software](https://www.nerves-project.org/) and making inroads into machine learning and data analysis with projects such as [Nx (Numerical Elixir)](https://github.com/elixir-nx/nx), [Explorer](https://github.com/elixir-nx/explorer), [Axon](https://github.com/elixir-nx/axon), and [Livebook](https://livebook.dev/) ([here is a summary of the main Numerical Elixir projects](https://github.com/elixir-nx/). Elixir was designed to be extensible and how it will be extended has always been a community effort. + +For these reasons, I choose to focus on *My Future* with Elixir. Those are the projects I am personally excited about and working on alongside other community members. The topic of today's article is type systems, as discussed in my ElixirConf EU presentation in May 2022. + +## The elephant in the room: types + +Throughout the years, the Elixir Core Team has addressed the biggest needs of the community. [Elixir v1.6 introduced the Elixir code formatter](https://elixir-lang.org/blog/2018/01/17/elixir-v1-6-0-released/), as the growing community and large teams saw an increased need for style guides and conventions around large codebases. + +[Elixir v1.9 shipped with built-in support for releases](https://elixir-lang.org/blog/2019/06/24/elixir-v1-9-0-released/): self-contained archives that consist of your application code, all of its dependencies, plus the whole Erlang Virtual Machine (VM) and runtime. The goal was to address the perceived difficulty in deploying Elixir projects, by bringing tried approaches from both Elixir and Erlang communities into the official tooling. This paved the way to future automation, such as `mix phx.gen.release`, which automatically generates a Dockerfile tailored to your Phoenix applications. + +Given our relationship with the community, it would be disingenuous to talk about my future with Elixir without addressing what seems to be the biggest community need nowadays: static typing. However, when the community asks for static typing, what are we effectively expecting? And what is the Elixir community to gain from it? + +## Types and Elixir + +Different programming languages and platforms extract different values from types. These values may or may not apply to Elixir. + +For example, different languages can extract performance benefits from types. However, Elixir still runs on the Erlang VM, which is dynamically typed, so we should not expect any meaningful performance gain from typing Elixir code. + +Another benefit of types is to _aid_ documentation (emphasis on the word _aid_ as I don't believe types replace textual documentation). Elixir already reaps similar benefits from [typespecs](https://hexdocs.pm/elixir/typespecs.html) and I would expect an integrated type system to be even more valuable in this area. + +However, the upsides and downsides of static typing become fuzzier and prone to exaggerations once we discuss them in the context of code maintenance, in particular when comparing types with other software verification techniques, such as tests. In those situations, it is common to hear unrealistic claims such as "a static type system would catch 80% of my Elixir bugs" or that "you need to write fewer tests once you have static types". + +While [I explore why I don't believe those claims are true during the keynote](https://www.youtube.com/watch?v=Jf5Hsa1KOc8), saying a static type system helps catch bugs is not helpful unless we discuss exactly the type of bugs it is supposed to identify, and that's what we should focus on. + +For example, Rust's type system helps prevent bugs such as deallocating memory twice, dangling pointers, data races in threads, and more. But adding such type system to Elixir would be unproductive because those are not bugs that we run into in the first place, as those properties are guaranteed by the garbage collector and the Erlang runtime. + +This brings another discussion point: a type system naturally restricts the amount of code we can write because, in order to prove certain properties about our code, certain styles have to be rejected. However, I would prefer to avoid restricting the expressive power of Elixir, because I am honestly quite happy with the language semantics (which we mostly inherited from Erlang). + +For Elixir, the benefit of a type system would revolve mostly around contracts. If function `caller(arg)` calls a function named `callee(arg)`, we want to guarantee that, as both these functions change over time, that `caller` is passing valid arguments into `callee` and that the `caller` properly handles the return types from `callee`. + +This may seem like a simple guarantee to provide, but we'd run into tricky scenarios even on small code samples. For example, imagine that we define a `negate` function, that negates numbers. One may implement it like this: + +```elixir +def negate(x) when is_integer(x), do: -x +``` + +We could then say `negate` has the type `integer() -> integer()`. + +With our custom negation in hand, we can implement a custom subtraction: + +```elixir +def subtract(a, b) when is_integer(a) and is_integer(b) do + a + negate(b) +end +``` + +This would all work and typecheck as expected, as we are only working with integers. However, imagine in the future someone decides to make `negate` polymorphic, so it also negates booleans: + +```elixir +def negate(x) when is_integer(x), do: -x +def negate(x) when is_boolean(x), do: not x +``` + +If we were to naively say that `negate` now has the type `integer() | boolean() -> integer() | boolean()`, we would now get a false positive warning in our implementation of subtract: + +```elixir +Type warning: + + | + | def subtract(a, b) when is_integer(a) and is_integer(b) do + | a + negate(b) + ^ the operator + expects integer(), integer() as arguments, + but the second argument can be integer() | boolean() +``` + +**So we want a type system that can type contracts between functions but, at the same time, avoids false positives and does not restrict the Elixir language**. Balancing those trade-offs is not only a technical challenge but also one that needs to consider the needs of the community. The [Dialyzer project](https://www.erlang.org/doc/man/dialyzer.html), implemented in Erlang and available for Elixir projects, chose to have no false positives. However, that implies certain bugs may not be caught. + +At this point in time, it seems the overall community would prefer a system that flags more potential bugs, even if it means more false positives. This may be particularly tricky in the context of Elixir and Erlang because I like to describe them as [_assertive languages_](https://dashbit.co/blog/writing-assertive-code-with-elixir): we write code that will crash in face of unexpected scenarios because we rely on supervisors to restart parts of our application whenever that happens. This is the foundation of building self-healing and fault-tolerant systems in those languages. + +On the other hand, this is what makes a type system for Erlang/Elixir so exciting and unique: the ability to deal with failure modes both at compile-time and runtime elegantly. Because at the end of the day, regardless of the type system of your choice, you will run into unexpected scenarios, especially when interacting with external resources such as the filesystem, APIs, distributed nodes, etc. + +## The big announcement + +This brings me to the big announcement from ElixirConf EU 2022: **we have an on-going PhD scholarship to research and develop a type system for Elixir based on set-theoretic types**. Guillaume Duboc (PhD student) is the recipient of the scholarship, lead by Giuseppe Castagna (Senior Resercher) with support from José Valim (that's me). + +The scholarship is a partnership between the [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). It is sponsored by Supabase ([they are hiring!](https://supabase.com/company)), Fresha ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), and [Dashbit](https://dashbit.co/), all heavily invested in Elixir's future. + +## Why set-theoretic types? + +We want a type system that can elegantly model all of Elixir idioms and, at a first glance, set-theoretic types were an excellent match. In set-theoretic types, we use set operations to define types and ensure that the types satisfy the associativity and distributivity properties of the corresponding set-theoretic operations. + +For example, numbers in Elixir can be integers _or_ floats, therefore we can write them as the union `integer() | float()` (which is equivalent to `float() | integer()`). + +Remember the `negate` function we wrote above? + +```elixir +def negate(x) when is_integer(x), do: -x +def negate(x) when is_boolean(x), do: not x +``` + +We could think of it as a function that has both types `(integer() -> integer())` and `(boolean() -> boolean())`, which is as an intersection. This would naturally solve the problem described in the previous section: when called with an integer, it can only return an integer. + +We also have a data-structure called atoms in Elixir. They uniquely represent a value which is given by their own name. Such as `:sunday` or `:banana`. You can think of the type `atom()` as the set of all atoms. In addition, we can think of the values `:sunday` and `:banana` as subtypes of `atom()`, as they are contained in the set of all atoms. `:sunday` and `:banana` are also known as singleton types (as they are made up of only one value). + +In fact, we could even consider each integer to be a singleton type that belongs to the `integer()` set. The choice of which values will become singletons in our type system will strongly depend on the trade-offs we defined in the previous sections. The type system also has to be gradual, as typed Elixir code may interact with untyped Elixir code and vice-versa. + +Personally, I find set-theoretical types an elegant and accessible approach to reason about types. At the end of the day, an Elixir developer won't have to think about intersections when writing a function with multiple clauses, but the modelling is straight-forward if they are ever to look under the hood. + +Despite the initial fit between Elixir semantics and set-theoretic types, there are open questions and existing challenges in putting the two together. Here are some examples: + + * Elixir has [an expressive collection of idioms used in pattern matching and guards](https://hexdocs.pm/elixir/patterns-and-guards.html), can we map them all to set-theoretic types? + + * Elixir associative data structures, [called maps](https://hexdocs.pm/elixir/Map.html), can be used both as records and as dictionaries. Would it be possible to also type them with a unified foundation? + + * Gradual type systems must introduce runtime type checks in order to remain sound. However, those type checks will happen in addition to the checks already done by the Erlang VM, which can degrade performance. Therefore, is it possible to leverage the existing runtime checks done by the Erlang VM so the resulting type system is still sound? + +Those challenges are precisely what makes me excited to work with Giuseppe Castagna and Guillaume Duboc, as we believe it is important to formalize those problems and their solutions, before we dig deep into the implementation. To get started with set-theoretic types, I recommend [Programming with union, intersection, and negation types by Giuseppe Castagna](https://www.irif.fr/~gc/papers/set-theoretic-types-2022.pdf). + +Finally, it is important to note there are areas we don't plan to tackle at the moment, such as typing of messages between processes. + +## Expectations and roadmap + +At this point, you may be expecting that Elixir will certainly become a gradually typed language at some moment in its future. However, it is important to note this may not be the case, as there is a long road ahead of us. + +One of the challenges in implementing a type system - at least for someone who doesn't have the relevant academic background like myself - is that it feels like a single indivisible step: you take a language without a type system and at the end you have one, without much insight or opportunity for feedback in the middle. Therefore we have been planning to incorporate the type system into Elixir in steps, which I have been referring to as "a gradual gradual type system": one where we add gradual types to the language gradually. + +The first step, the one we are currently working on, is to leverage the existing type information found in Elixir programs. As previously mentioned, [we write assertive code](https://dashbit.co/blog/writing-assertive-code-with-elixir) in Elixir, which means there is a lot of type information in patterns and guards. We want to lift this information and use it to type check existing codebases. The Erlang compiler already does so to improve performance within a single module and we want to eventually do so across modules and applications too. + +During this phase, Elixir developers won't have to change a single line of code to leverage the benefits of the type system. Of course, we will catch only part of existing bugs, but this will allows us to stress test, benchmark, and collect feedback from developers, making improvements behind the scenes (or even revert the whole thing if we believe it won't take us where we expect). + +The next step is to introduce typed structs into the language, allowing struct types to propagate throughout the system, as you pattern match on structs throughout the codebase. In this stage we will introduce a new API for defining structs, yet to be discussed, and developers will have to use the new API to reap its benefits. + +Finally, once we are happy with the improvements and the feedback collected, we can migrate to introduce a new syntax for typing function signatures in Elixir codebases, including support for more advanced features such as polymorphic types. Those will allow us to type complex constructs such as the ones found in the `Enum` module. + +The important point to keep in mind is that those features will be explored and developed in steps, with plenty of opportunity to gather community feedback. I also hope our experience may be useful to other ecosystems who wish to gradually introduce type systems into existing programming languages, in a way that feels granular and participative. + +Thank you for reading and see you in future updates. diff --git a/_posts/2022-12-22-cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience.markdown b/_posts/2022-12-22-cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience.markdown new file mode 100644 index 000000000..505bf15bf --- /dev/null +++ b/_posts/2022-12-22-cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience.markdown @@ -0,0 +1,188 @@ +--- +layout: post +title: "Cheatsheets and other 8 ExDoc features that improve the developer experience" +authors: +- Hugo Baraúna +category: Announcements +excerpt: This post explains the motivation behind the new ExDoc Cheatsheet feature. It also highlights other ExDoc features that show how ExDoc has been evolving to make the documentation experience in Elixir better and better. +--- + +ExDoc has a cool new feature, [cheatsheets](https://hexdocs.pm/ex_doc/cheatsheet.html)! + +In this blog post, we'll explain what that new feature is and the motivation behind it. We'll also take the opportunity to highlight other ExDoc features that show how it has been evolving to make the documentation experience in Elixir better and better. + +## What are ExDoc cheatsheets and how they improve the documentation experience + +ExDoc's cheatsheets are Markdown files with the `.cheatmd` extension. You can see [an example](https://hexdocs.pm/ecto/crud.html) of how the Ecto project is using them. + +Writing and reading cheatsheets is not exactly new to developers. What ExDoc brings to the table is the possibility of integrating cheatsheets alongside the rest of the documentation of an Elixir project, instead of hosting them in a different place. + +Developers need different kinds of documentation at different times. When one is learning about a new library, a guide format is proper. When one needs to know if a library can solve a specific problem, an API reference can be more appropriate. When someone wants to remember a couple of functions they already used from that library, a cheatsheet could be more practical. + +Imagine if you had to go to a different place for every type of documentation you're looking for. That would make a very fragmented experience, not only for readers of documentation but also for writers. + +ExDoc cheatsheets represent one step further in the direction of making documentation in Elixir an even more comprehensive and integrated experience. + +ExDoc cheatsheets are inspired by [devhints.io](https://devhints.io) from [Rico Sta. Cruz](https://twitter.com/rstacruz), and were contributed by [Paulo Valim](https://twitter.com/paulovalim) and [Yordis Prieto](https://twitter.com/alchemist_ubi). + +## Eight features that show how ExDoc has improved developer experience over time + +We added cheatsheets to ExDoc because we value developer experience and believe documentation is a core aspect of it. + +Since the beginning, one of Elixir's principles is that documentation should be a first-class citizen. What this idea means to us is that documentation should be easy to write and easy to read. ExDoc has been continuously evolving over the years, guided by this principle. + +Here are some of the features added to ExDoc over the years that make reading and writing documentation in Elixir a joy. + +### Beautiful and usable design + +As developers, we may not have the skill to make beautifully designed UIs. That doesn't mean we don't appreciate it. Here's what documentation generated with ExDoc looked like almost ten years ago, with its original layout based on [YARD](https://yardoc.org/): + +![Screenshot of the Phoenix v0.5.0 documentation generated with an early version +of ExDoc](https://i.imgur.com/O9xKjR8.jpg) + +Here's what it looks like today: + +![Screenshot of the Phoenix v1.6.15 documentation generated with current +ExDoc](https://i.imgur.com/ZKI1T23.png) + +The evolution of ExDoc's design helped documentation be more visually appealing and easier to read and navigate. + +### Links to source code + +Sometimes you're reading the documentation of a library, and you want to know more about the implementation of a function. Or, you found something in the documentation that could be improved and wants to help. In those situations, it's helpful to go from the documentation to the source code. ExDoc makes that dead easy. For every module, function, or page, ExDoc gives you a link that you can click to go directly to the project's source code on GitHub: + +![Short screencast of a user clicking on the "link to source code" button on the +documentation for a function](https://i.imgur.com/PXvoeDk.gif) + +### Guides + +One of the most common formats of library documentation is an API reference. But depending on your needs, that's not the most approachable format. For example, it's not optimal when you're just getting started with a library or when you want to learn how to solve a specific problem using it. That's why ExDoc allows writing other types of docs besides API references, like *"Getting started" guides* or *How-tos*. + +Look at how [Ecto's documentation](https://hexdocs.pm/ecto/getting-started.html) uses that, for example: + +![Screencast of a user exploring the guides in the Ecto +documentation](https://i.imgur.com/KInZb4x.gif) + +### Custom grouping of modules, functions, and pages in the sidebar + +Sometimes your library has dozens of modules. Sometimes, one given module has a large API surface area. In those situations, showing the list of functions as a single large list may not be the most digestible way to be consumed. For those cases, ExDoc allows modules, functions, or extra pages to be grouped in the sidebar in a way that makes more sense semantically. + +Here's an example of how Ecto use grouped functions for its `Repo` module: + +![Screenshot of the sidebar of the Ecto documentation, showing grouped functions +in the `Ecto.Repo` module](https://i.imgur.com/ZE7N312.png) + +Instead of listing the ~40 functions of `Ecto.Repo` as a single extensive list, it presents them grouped by five cohesive topics: + +- Query API +- Schema API +- Transaction API +- Runtime API +- User callbacks + +The same functionality is available for modules and pages (guides, how-tos, and so on). Phoenix is a [good example](https://hexdocs.pm/phoenix/overview.html) of how that's used. + +### Full-text search + +Sometimes you don't know or don't remember the name of the function that you're looking for. For example, let's say you're looking for a function for dealing with file system directories. + +Although there's no function or module called "directory" in Elixir, when you type "directory" in [Elixir's documentation](https://hexdocs.pm/elixir/search.html?q=directory), it will return all the entries that have the word "directory" inside the documentation. It will even return entries with variations of the word "directory", like "directories", doing a fuzzy search. + +![Screenshot of the result of searching for "directory" in the Elixir +documentation](https://i.imgur.com/IHHuej8.png) + +The search bar also supports autocompletion for module and function names: + +![Screencast of a user typing the word "Enum" in the search bar of Elixir's +documentation and letting it autocomplete the module. Then, the user types +"Range" and both modules and functions show +up.](https://i.imgur.com/2cmsuDi.gif) + +The best part is that full-text search is fully implemented on the client-side, which means ExDoc pages can be fully hosted as static websites (for example on GitHub Pages). + +### Keyboard shortcuts to navigate to docs of other Hex packages + +It's common for an application to have dependencies. While coding, we usually need to read the documentation of more than one of those dependencies. + +One solution is to keep a window open for the documentation of each dependency. However, ExDoc offers another option: a keyboard shortcut to search and go to another package documentation within the same window. + +Here's what it looks like: + +![Screencast of a user enabling the `g` shortcut to search through dependencies +documentation and then using it to search for "phoenix_live" in the +documentation for Nerves.](https://i.imgur.com/I9uJxUF.gif) + +There are more keyboard shortcuts to help you navigate within and between documentation: + +![Screenshot of the keyboard shortcuts that you can enable in +ExDoc](https://i.imgur.com/qdoNUx9.png) + +### A version dropdown to switch to other versions + +Keeping our application updated with the latest versions of all its dependencies can be challenging. So, it's common to need to look at the documentation of an older version of a library we're using. ExDoc makes it simple to do that. + +When you access the documentation of a project, there's a dropdown that you can use to select the version you're looking for: + +![Screencast of a user typing the version dropdown under the application name in +the "timex" documentation, revealing all the +versions.](https://i.imgur.com/1krcY5g.gif) + +### Livebook integration + +[Livebook](https://livebook.dev/) is a web application for writing interactive and collaborative code notebooks in Elixir. + +One of the ways Elixir developers have been using Livebook is for documentation. Because of its interactivity capabilities, it enables the reader to play with the code right inside the documentation, which makes it great for tutorials and augmenting the user experience. + +With that in mind, ExDoc offers the possibility of integrating Livebook notebooks. That means one can host Livebook-based documentation together with the API reference. + +Here's an [example of using Livebook inside ExDoc for writing a Usage Guide](https://hexdocs.pm/req_sandbox/usage.html): + +![Screencast of a user navigating through the "req_sandbox" documentation, +finding a Livebook, clicking "Run in Livebook", and using the Livebook that +opens up on their local machine.](https://i.imgur.com/FxOLs0Y.gif) + +### Bonus: Erlang support + +[EEP 48](https://www.erlang.org/eeps/eep-0048) proposed a standardized way for how BEAM languages could store API documentation. This allows any BEAM language to read documentation generated by each other. + +By leveraging that work, ExDoc can generate documentation for an Erlang project. For example, Telemetry is a library written in Erlang that has [its documentation](https://hexdocs.pm/telemetry/readme.html) generated with ExDoc. + +![Screenshot of "telemetry" documentation generated with +ExDoc](https://i.imgur.com/C4Idbuh.png) + +By using ExDoc to also generate documentation for Erlang-based projects, we can have more consistency in the user experience along the BEAM ecosystem. See the great [`rebar3_ex_doc`](https://hexdocs.pm/rebar3_ex_doc/) plugin to get started. + +### Bonus: Doctests + +When writing documentation, it's helpful to offer code examples. For instance, here's the documentation of the `Enum.any?/1` function from Elixir's standard library: + +```elixir +@doc """ +Returns `true` if at least one element in `enumerable` is truthy. + +When an element has a truthy value (neither `false` nor `nil`) iteration stops +immediately and `true` is returned. In all other cases `false` is returned. + +## Examples + + iex> Enum.any?([false, false, false]) + false + + iex> Enum.any?([false, true, false]) + true + + iex> Enum.any?([]) + false + +""" +``` + +To ensure examples do not get out of date, Elixir's test framework ExUnit provides a feature called **doctests**. This allows developers to test the examples in their documentation. Doctests work by parsing out code samples starting with `iex>` from the documentation. + +Although this is not a feature of ExDoc, it is an essential part of Elixir's developer and documentation experience. + +## Wrap up + +As we saw, ExDoc has evolved a lot throughout the years! As it continues to evolve into a more and more comprehensive documentation tool, we want to enable developers to keep investing more time writing the documentation itself instead of needing to spend time building custom documentation tools and websites. The best part is that all you need to do to leverage many of those features is to simply document your code using the `@doc` attribute! + +Here's to a continuously improving documentation experience for the next years. diff --git a/_posts/2023-03-09-embedded-and-cloud-elixir-at-sparkmeter.markdown b/_posts/2023-03-09-embedded-and-cloud-elixir-at-sparkmeter.markdown new file mode 100644 index 000000000..2a5fa73ce --- /dev/null +++ b/_posts/2023-03-09-embedded-and-cloud-elixir-at-sparkmeter.markdown @@ -0,0 +1,125 @@ +--- +layout: post +title: "Embedded and cloud Elixir for grid-management at Sparkmeter" +authors: +- Hugo Baraúna +category: Elixir in Production +excerpt: A case study of how Elixir is being used at SparkMeter. +logo: /images/cases/logos/sparkmeter.png +tags: energy iot nerves +--- + +*Welcome to our series of case studies about companies using Elixir in production. [See all](https://elixir-lang.org/cases.html) cases we have published so far.* + +[SparkMeter](https://www.sparkmeter.io/) is a company on a mission to increase access to electricity. They offer grid-management solutions that enable utilities in emerging markets to run financially-sustainable efficient, and reliable systems. + +Elixir has played an important role in simplifying SparkMeter systems by providing a unified developer experience across their products. Elixir's versatility in different domains, such as embedded software, data processing, and HTTP APIs, proved to be a valuable asset to a team who aims to release robust products quickly and confidently. + +Two of their products are smart electrical meters and grid-management software. These can be used to measure electricity usage, gather health information about an electrical grid, and manage billing. + +Here's an overview of their architecture: + +![SparkMeter architecture generation one](/images/cases/bg/sparkmeter-old-architecture.png) + +The meters are embedded devices responsible for collecting measures such as electricity usage. They communicate with each other via a mesh network and also communicate with the grid edge management unit. The grid edge management unit is an embedded system that receives and processes data from up to thousands of meters. The grid edge management unit also communicates with servers running in the cloud. Those servers send and receive data to the grid edge management units and process it for use by internal systems and user-facing software. + +## The challenge + +The infrastructure in which their embedded devices are deployed is not reliable. The cellular network used for communication between the ground and the cloud could fail, and the electricity supply to the embedded systems could go down. Therefore, their system needed to be fault-tolerant, and they needed to build equipment that didn't require constant field maintenance. + +In light of these requirements, they identified areas for improvement in the first generation of their product. One of the things they needed to improve was the development of a new grid edge management unit. Additionally, their product was mission-critical, so they wanted a technology they could confidently put into production and one that would not take more than a year of development and QA before releasing a new generation of their product. + +That's when they discovered Elixir and Nerves. + +## The trade-offs of adopting Elixir and Nerves +Nerves is an open-source platform that combines the Erlang virtual machine and Elixir ecosystem to build and deploy embedded systems. + +When considering the adoption of Elixir and [Nerves](https://nerves-project.org/), SparkMeter recognized many advantages the technologies offered. + +Elixir helped them meet the requirement of building a distributed and fault-tolerant system. That's because Elixir leverages the power of the Erlang VM and the OTP framework, which were designed with that requirement in mind. + +Regarding Nerves, they saw it as an entire ecosystem for doing embedded development with many advantages. For example, it has a good story for doing local development and going from that to [deploying](https://www.nerves-hub.org/) on an embedded device. It makes it easy to connect to an embedded device for iterative development. And it also enables fine-grained control of system boot, so they can handle scenarios when certain parts of the system won't start. + +That said, they had two concerns, the growth of Nerves and finding talent with expertise in the Elixir/Nerves stack. + +They wanted to ensure that Nerves would continue to grow. But they realized that even if it didn't, the benefits Nerves was already offering could give them a lot of leverage. Here are's what their senior VP of engineering, Jon Thacker, had to say about that: + +> Without Nerves, we would be on our own to figure out a lot. How to do distribution, the development environment, and how to support different architectures. So it really is a batteries-included framework for doing production-grade embedded systems. +> +> \- *Jon Thacker, Senior VP of Engineering* + +When we interviewed Jon for this case study, they had already been using Elixir and Nerves for more than two years. And with the benefit of hindsight, here's what he said about adopting Nerves: + +> Making sure that Nerves continued to grow was a concern. But it has done so and is showing a very positive trajectory. It was a calculated risk and, as it turns out, it was the correct choice. +> +> \- *Jon Thacker, Senior VP of Engineering* + +When it came to finding talent, they approached the problem in two ways. First, they started to build the pilot with a contractor to ensure that the staffing risk didn't affect their timeline. But they also wanted to have an internal team to take ownership of the product in the long term. So, shortly after finishing the first version of the new system, they hired two engineers with experience in Elixir, Michael Waud and Benjamin Milde. + +Besides hiring people with previous experience in Elixir, Jon noticed that training their embedded engineers in Elixir was also a viable option. Here's what he told us about that: + +> I'm traditionally an embedded engineer, and I only learned Elixir as part of this project. However, transferring my mental model was so easy that I do believe that we would be capable of training other embedded engineers as well. +> +> \- *Jon Thacker, Senior VP of Engineering* + +## The new system + +SparkMeter used Elixir for the ground (embedded) and cloud aspects of the new system they built. Here is an overview of the architecture: + +![SparkMeter architecture generation two](/images/cases/bg/sparkmeter-new-architecture.png) + +For the firmware of the grid edge management unit, they used Nerves. For the hardware, they built on top of a BeagleBone Black device. + +The communication between the grid edge management unit and the meters was via radio, using Rust to manage the radio hardware module inside the grid edge management unit. They used [Elixir Ports](https://hexdocs.pm/elixir/1.13.4/Port.html) to communicate with Rust and process the data from the meters. + +Elixir was also used for communication with the cloud servers via 3G or Edge. This communication required bandwidth usage optimization due to the cost of sending large volumes of data through the cellular network. They evaluated various solutions like REST, CoAP, MQTT, Kafka, and Websockets. Still, none fit their specific needs, so they created a custom protocol tailored to their use case, which involved designing a binary protocol and implementing a TCP server. Mike Waud discussed this in more detail in his talks at [ElixirConf 2021](https://www.youtube.com/watch?v=DJRL86mO4ks) and [2022](https://www.youtube.com/watch?v=BxTIUvyZHKw). + +The grid edge management unit also required a local web user interface that could be accessed on-site via Wi-Fi. For this, they used Phoenix and Liveview. + +The cloud aspect of the system is responsible for receiving data from the grid edge management units and sending control commands. It also runs a TCP server with their custom protocol, implemented in Elixir. The data received from the grid edge management units is stored in PostgreSQL and then consumed by a [Broadway-based](https://elixir-broadway.org/) data pipeline. + +The cloud system also exposes an HTTP API implemented with Phoenix. This API is consumed by other internal systems to interact with their PostgreSQL database. + +## Reaping the benefits + +During and after the development of the new generation of their system, SparkMeter observed many benefits. + +One of them was the reduction of the complexity of the grid edge management unit. The old version had more moving parts, using Ubuntu and Docker for the system level, Python/Celery and RabbitMQ for asynchronous processing, and Systemd for managing starting job processes. + +In the new version, they replaced all of that mainly with Elixir and Nerves. And for the parts where they needed tools that were not part of the BEAM stack, they could manage them like any other BEAM process by using [Elixir Ports](https://hexdocs.pm/elixir/1.13.4/Port.html). Here's what they said about that experience: + +> The new grid edge management unit has a very unified architecture. We can treat everything as an (Elixir) process. We have full control over the start and stop within a single ecosystem. It's just a very coherent storyline. +> +> \- *Jon Thacker, Senior VP Of Engineering* + +Another aspect they liked about Nerves was that it included security best practices. For example, they used SSL certificates on the client and the server side for communication between the ground and the cloud. Nerves made this easy through the [NervesKey component](https://github.com/nerves-hub/nerves_key), which enables the use of a hardware security module to protect the private key. Nerves also made it easy to keep up with system security patches, as the firmware generated by Nerves is a single bundle containing a minimal Linux platform and their application packaged as a [release](https://hexdocs.pm/mix/Mix.Tasks.Release.html). Here's what they said about security in Nerves: + +> It's easy enough to keep tracking upstream changes, so we're not getting behind the latest security patches. Nerves made that easy. Nerves just pushed us towards a good security model. +> +> \- *Jon Thacker, Senior VP Of Engineering* + +The communication between the ground and the cloud involved implementing a custom TCP server running in both parts of the system. Network programming is not an everyday task for many application developers, but Elixir helped them a lot with that: + +> I had never written a TCP client or a server before, it's just not something you even think about. But doing it in Elixir, particularly on the protocol level of sending binaries, was a pleasure to work with! Something that would be super tedious in an imperative language, with Elixir and pattern matching, is so clear! +> +> \- *Michael Waud, Senior Software Engineer* + +Another benefit they received from using Elixir on the ground and in the cloud was code reuse. For example, the encoding and decoding of their custom protocol were reused for both the embedded and cloud parts. + +> It would've been a much larger challenge if we hadn't been running Elixir in the cloud and on the grid edge management unit because we could write it once. The encoding and decoding we wrote once, we gained a lot from being able to share code. +> +> \- *Michael Waud, Senior Software Engineer* + +Michael also pointed out that by controlling the complete connection from the grid edge management unit up to the cloud, they could reduce bandwidth usage and improve resiliency, which were essential requirements for them. + +Finally, the new generation of their system also enabled them to release more often. Before, they were releasing new versions every quarter, but with the new system, they could release weekly when needed. + +## Summing up + +In conclusion, SparkMeter's adoption of Elixir and Nerves has led to many benefits for their mission-critical grid-management system. + +Elixir was used to design elegant solutions across data processing, HTTP APIs, and within the embedded space. This unified development model led to a more productive and robust environment, with less complexity and fewer moving parts. + +Additionally, the ability to control the entire connection from the ground to the cloud resulted in reduced bandwidth usage and improved resiliency. This fulfills essential requirements, given the diversity of conditions and locations the grid edge management unit may be deployed at. + +The new system also allowed for more frequent releases, enabling SparkMeter to respond quickly to their business needs. \ No newline at end of file diff --git a/_posts/2023-06-19-elixir-v1-15-0-released.markdown b/_posts/2023-06-19-elixir-v1-15-0-released.markdown new file mode 100644 index 000000000..ffc23b810 --- /dev/null +++ b/_posts/2023-06-19-elixir-v1-15-0-released.markdown @@ -0,0 +1,170 @@ +--- +layout: post +title: Elixir v1.15 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.15 with improvements to compilation and boot times. +--- + +Elixir v1.15 has just been released. 🎉 + +Elixir v1.15 is a smaller release with focused improvements +on compilation and boot times. This release also completes +our integration process with Erlang/OTP logger, bringing new +features such as log rotation and compression out of the box. + +You will also find additional convenience functions in `Code`, +`Map`, `Keyword`, all Calendar modules, and others. + +Finally, we are glad to welcome [Jean Klingler](https://github.com/sabiwara/) +as a member of the Elixir Core team. Thank you for your contributions! + +## Compile and boot-time improvements + +The last several releases brought improvements to compilation +time and this version is no different. In particular, Elixir +now caches and prunes load paths before compilation, ensuring your +project (and dependencies!) compile faster and in an environment +closer to production. + +In a nutshell, the Erlang VM loads modules from code paths. Each +application that ships with Erlang and Elixir plus each dependency +become an entry in your code path. The larger the code path, the +more work Erlang has to do in order to find a module. + +In previous versions, Mix would only add entries to the load paths. +Therefore, if you compiled 20 dependencies and you went to compile +the 21st, the code path would have 21 entries (plus all Erlang and +Elixir apps). This allowed modules from unrelated dependencies to +be seen and made compilation slower the more dependencies you had. +With this release, we will now prune the code paths to only the ones +listed as dependencies, bringing the behaviour closer to `mix release`. + +Furthermore, Erlang/OTP 26 allows us to start applications +concurrently and cache the code path lookups, decreasing the cost of +booting applications. The combination of Elixir v1.15 and Erlang/OTP 26 +should also reduce the boot time of applications, such as when starting +`iex -S mix` or running a single test with `mix test`. + +As an example, I have benchmarked [the Livebook application](https://github.com/livebook-dev/livebook) +on a M1 Max MacStudio across different Elixir and Erlang/OTP versions. +At the time of benchmarking, Livebook had ~200 source `.ex` files and +~35 dependencies. Compilation-times were improved by 16%: + +![Livebook compilation times](/images/contents/livebook-compile-1.15.png) + +Livebook saw an improvement of 30% on boot times: + +![Livebook boot times](/images/contents/livebook-boot-1.15.png) + +Different application will see different results. Our expectations +are the gains will be more meaningful the more dependencies you have, +the more files you have, and the more cores you have. We have even +received reports of up to 40% faster compilation times, although it +is yet unclear how generalizable this will be in practice. Note this +work does not improve the time to compile slow individual files. + +The compiler is also smarter in several ways: `@behaviour` declarations +no longer add compile-time dependencies and aliases in patterns and +guards add no dependency whatsoever, as no dispatching happens. Furthermore, +Mix now tracks the digests of `@external_resource` files, reducing the +amount of recompilation when swapping branches. Finally, dependencies +are automatically recompiled when their compile-time configuration changes, +providing a smoother development experience. + +##### Potential incompatibilities + +Due to the code path pruning, if you have an application or dependency +that does not specify its dependencies on Erlang/OTP and core Elixir applications, +which has always been erroneus behaviour, it may no longer compile +successfully in Elixir v1.15. You can temporarily disable code path pruning +by setting `prune_code_paths: false` in your `mix.exs`, although doing so +may lead to runtime bugs that are only manifested inside a `mix release`. + +## Compiler warnings and errors + +The Elixir compiler can now emit many errors for a single file, making +sure more feedback is reported to developers before compilation is aborted. + +In Elixir v1.14, an undefined function would be reported as: + + ** (CompileError) undefined function foo/0 (there is no such import) + my_file.exs:1 + +In Elixir v1.15, the new reports will look like: + + error: undefined function foo/0 (there is no such import) + my_file.exs:1 + + ** (CompileError) my_file.exs: cannot compile file (errors have been logged) + +A new function, called `Code.with_diagnostics/2`, has been added so this +information can be leveraged by editors, allowing them to point to several +errors at once. We have currently ongoing work and contribution to further +improve the compiler diagnostics in future Elixir releases. + +##### Potential incompatibilities + +As part of this effort, the behaviour where undefined variables were transformed +into nullary function calls, often leading to confusing error reports, has +been disabled during project compilation. You can invoke `Code.compiler_options(on_undefined_variable: :warn)` +at the top of your `mix.exs` to bring the old behaviour back. + +## Integration with Erlang/OTP logger + +This release provides additional features such as global logger +metadata and [file logging](https://hexdocs.pm/logger/Logger.html#module-erlang-otp-handlers) (with rotation and compression) out of the box! + +This release also soft-deprecates Elixir's Logger Backends in +favor of Erlang's Logger handlers. Elixir will automatically +convert your `:console` backend configuration into the new +configuration. Previously, you would set: + +```elixir +config :logger, :console, + level: :error, + format: "$time $message $metadata" +``` + +Which is now translated to the equivalent: + +```elixir +config :logger, :default_handler, + level: :error + +config :logger, :default_formatter, + format: "$time $message $metadata" +``` + +To replace the default console handler by one that writes to disk, +with log rotation and compression: + +```elixir +config :logger, :default_handler, + config: [ + file: ~c"system.log", + filesync_repeat_interval: 5000, + file_check: 5000, + max_no_bytes: 10_000_000, + max_no_files: 5, + compress_on_rotate: true + ] +``` + +Finally, the previous Logger Backends API is now soft-deprecated. +If you implement your own backends, you want to consider migrating to +[`:logger_backends`](https://github.com/elixir-lang/logger_backends) +in the long term. See the new [`Logger`](https://hexdocs.pm/logger) +documentation for more information on the new features and compatibility. + +## Learn more + +For a complete list of all changes, see the +[full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.15.0). + +Check [the Install section](/install.html) to get Elixir installed and +read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +to learn more. + +Happy compiling! diff --git a/_posts/2023-06-22-type-system-updates-research-dev.markdown b/_posts/2023-06-22-type-system-updates-research-dev.markdown new file mode 100644 index 000000000..1765f7830 --- /dev/null +++ b/_posts/2023-06-22-type-system-updates-research-dev.markdown @@ -0,0 +1,85 @@ +--- +layout: post +title: "Type system updates: moving from research into development" +authors: +- José Valim +category: Announcements +excerpt: A short status update on the effort to bring a type system into Elixir. +--- + +A year ago, at ElixirConf EU 2022, we announced an effort to research +and develop a type system for Elixir ([video presentation](https://www.youtube.com/watch?v=Jf5Hsa1KOc8)) +([written report](/blog/2022/10/05/my-future-with-elixir-set-theoretic-types/)). + +This work is happening under the lead of [Giuseppe Castagna](https://www.irif.fr/~gc/), +CNRS Senior Researcher, and taken by +[Guillaume Duboc](https://www.irif.fr/users/gduboc/index) as part of his +PhD studies, with further guidance from myself (José Valim). + +This article is a summary of where we are in our efforts and where we +are going. + +## Out of research + +Our main goal during research is to find a type system that can model +most of Elixir's functional semantics and develop brand new theory on +the areas we found to be incompatible or lacking. We believe we were +able to achieve this goal with a gradual set-theoretic type system +and we are now ready to head towards development. Over the last 2 months, +we have published plenty of resources on our results: + + * [A technical report on the design principles of the Elixir type system](https://arxiv.org/abs/2306.06391) + * [A technical presentation by Guillaume Duboc at ElixirConf 2023 on the work above](https://youtube.com/watch?v=gJJH7a2J9O8) + * [An informal discussion with Giuseppe Castagna, Guillaume Duboc, and José Valim on the SmartLogic podcast](https://smartlogic.io/podcast/elixir-wizards/s10-e12-jose-guillaume-giuseppe-types-elixir/) + * [An informal Q&A with Guillaume Duboc, José Valim, and the community on Twitch](https://www.twitch.tv/videos/1841707383) + +Our focus so far has been on the semantics. While we have introduced a +new syntax capable of expressing the semantics of the new set-theoretic +type system, the syntax is not final as there are still no concrete +plans for user-facing changes to the language. Once we are confident +those changes will happen, we will have plenty of discussion with the +community about the type system interface and its syntax. + +The work so far has been made possible thanks to a partnership between +the [CNRS](https://www.cnrs.fr/fr) and [Remote](https://remote.com), +with sponsorships from [Fresha](https://www.fresha.com), +[Supabase](https://supabase.com), and [Dashbit](https://dashbit.co). + +## Into development + +While there is still on-going research, our focus for the second semester +of 2023 onwards is on development. + +Incorporating a type system into a language used at scale can be a daunting +task. Our concerns range from how the community will interact and use the +type system to how it will perform on large codebases. Therefore, our plan +is to gradually introduce our gradual (pun intended) type system into the +Elixir compiler. + +In the first release, types will be used just internally by the compiler. +The type system will extract type information from patterns and guards to +find the most obvious mistakes, such as typos in field names or type +mismatches from attempting to add an integer to a string, without introducing +any user-facing changes to the language. At this stage, our main goal is +to assess the performance impact of the type system and the quality of +the reports we can generate in case of typing violations. If we are +unhappy with the results, we still have time to reassess our work or drop +the initiative altogether. + +The second milestone is to introduce type annotations only in structs, +which are named and statically-defined in Elixir codebases. Elixir programs +frequently pattern match on structs, which reveals information about +the struct fields, but it knows nothing about their respective types. +By propagating types from structs and their fields throughout the program, +we will increase the type system’s ability to find errors while further +straining our type system implementation. + +The third milestone is to introduce the (most likely) `$`-prefixed type +annotations for functions, with no or very limited type reconstruction: +users can annotate their code with types, but any untyped parameter +will be assumed to be of the `dynamic()` type. If successful, then we +will effectively have introduced a type system into the language. + +This new exciting development stage is sponsored by [Fresha](https://www.fresha.com) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team) ([they are hiring!](https://starfish.team/jobs/experienced-elixir-developer)), +and [Dashbit](https://dashbit.co). diff --git a/_posts/2023-09-20-strong-arrows-gradual-typing.markdown b/_posts/2023-09-20-strong-arrows-gradual-typing.markdown new file mode 100644 index 000000000..cd62f8c33 --- /dev/null +++ b/_posts/2023-09-20-strong-arrows-gradual-typing.markdown @@ -0,0 +1,293 @@ +--- +layout: post +title: "Strong arrows: a new approach to gradual typing" +authors: +- José Valim +category: Announcements +excerpt: An introduction to strong arrows and how it leverages the Erlang VM to provide sound gradual typing. +--- + +*This is article expands on the topic of gradual set-theoretic typing discussed during my keynote at [ElixirConf US 2023](https://www.youtube.com/watch?v=giYbq4HmfGA).* + +There is an on-going effort [to research and develop a type system for Elixir](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/), lead by [Giuseppe Castagna](https://www.irif.fr/~gc/), CNRS Senior Researcher, and taken by [Guillaume Duboc](https://www.irif.fr/users/gduboc/index) as part of his PhD studies. + +In this article, we will discuss how the proposed type system will tackle gradual typing and how it relates to set-theoretic types, with the goal of providing an introduction to the ideas [presented in our paper](https://arxiv.org/abs/2306.06391). + +## Set-theoretic types + +The type system we are currently researching and developing for Elixir is based on set-theoretic types, which is to say its operations are based on the fundamental set operations of union, intersection, and negation. + +For example, the atom `:ok` is a value in Elixir, that can be represented by the type `:ok`. All atoms in Elixir are represented by themselves in the type system. A function that returns either `:ok` or `:error` is said to return `:ok or :error`, where the `or` operator represents the union. + +The types `:ok` and `:error` are contained by the type `atom()`, which is an infinite set representing all atoms. The union of the types `:ok` and `atom()` can be written as `:ok or atom()`, and is equivalent to `atom()` (as `:ok` is a subset of `atom()`). The intersection of the types `:ok` and `atom()` can be written as `:ok and atom()`, and is equivalent to `:ok`. + +Similarly, `integer()` is another infinite set representing all integers. `integer() or atom()` is the union of all integers and atoms. The intersection `integer() and atom()` is an empty set, which we call `none()`. The union of all types that exist in Elixir is called `term()`. + +The beauty of set-theoretic types is that we can model many interesting properties found in Elixir programs on top of those fundamental set operations, which in turn we hope to make typing in Elixir both more expressive and accessible. Let's see an example of how a type system feature, called bounded quantification (or bounded polymorphism), can be implemented with set-theoretic types. + +## Upper and lower bounds + +The `identity` function is a function that receives an argument and returns it as is. In Java, it would be written as follows: + +```java +static T identity(T arg) { + return arg; +} +``` + +In TypeScript: + +```typescript +function identity(arg: T): T { + return arg; +} +``` + +Or in Haskell: + +```haskell +id :: a -> a +id arg = arg +``` + +In all of the examples above, we say the function receives an argument of type variable `T` (or type variable `a` in Haskell's case) and return a value of the same type `T`. We call this parametric polymorphism, because the function parameter - its argument - can take many (poly) shapes (morphs). In Elixir, we could then support: + +```elixir +$ a -> a +def identity(arg), do: arg +``` + +Sometimes we may want to further constrain those type variables. As example, let's constraint the identity function in Java to numbers: + +```java +static T identity(T arg) { + return arg; +} +``` + +Or in TypeScript: + +```typescript +function identity(arg: T): T { + return arg; +} +``` + +In Haskell, we can constrain to a typeclass, such as `Ord`: + +```haskell +id :: Ord a => a -> a +id x = x +``` + +In other words, these functions can accept any type as long as they fulfill a given constraint. This in turn is called bounded polymorphism, because we are putting bounds on the types we can receive. + +With all that said, how can we implement bounded polymorphism in set-theoretic types? Imagine we have a type variable `a`, how can we ensure it is bounded or constrained to another type? + +With set-theoretic types, this operation is an intersection. If you have `a and atom()`, `a` can be the type `:foo`. `a` can also be the type `atom()`, which represents all atom types, but `a` cannot be `integer()`, as `integer() and atom()` will return an empty set. In other words, there is no need to introduce a new semantic construct, as intersections can be used to place upper bounds in type variables! Therefore, we could restrict Elixir's identity function to numbers like this: + +```elixir +$ a and number() -> a and number() +def identity(arg), do: arg +``` + +Of course, we can provide syntax sugar for those constraints: + +```elixir +$ a -> a when a: number() +def identity(arg), do: arg +``` + +But at the end of the day it will simply expand to intersections. The important bit is that, at the semantic level, there is no need for additional constructs and representations. + +> Note: for the type-curious readers, set-theoretic types implement [a limited form of bounded quantification *à la* Kernel Fun](http://lucacardelli.name/Papers/OnUnderstanding.pdf). In a nutshell, it means we can only compare functions if they have the same bounds. For example, our type system states `a -> a when a: integer() or boolean()` is not a subtype of `a -> a when a: integer()`. + +We also get lower bounds for free. If intersections allow us to place an upper bound on a type variable, a union is equivalent to a lower bound as it specifies the type variable will always be augmented by the union-ed type. For example, `a or atom()` says the result will always include atoms plus whatever else specified by `a` (which may be an atom, `atom()` itself, or a completely disjoint type such as `integer()`). + +Elixir protocols, which is an Elixir construct equivalent to Haskell Typeclasses or Java interfaces, is another example of functionality that can be modelled and composed with set-theoretic types without additional semantics. The exact mechanism to do so is left as an exercise to the reader (or the topic of a future blog post). + +## Enter gradual typing + +Elixir is a functional dynamic programming language. Existing Elixir programs are untyped, which means that a type system needs mechanisms to interface existing Elixir code with future statically typed Elixir code. We can achieve this with gradual typing. + +A gradual type system is a type system that defines a `dynamic()` type. It is sometimes written as `?` and sometimes known as the `any` type (but I prefer to avoid `any` because it is too short and too lax in languages like TypeScript). + +In Elixir, the `dynamic()` type means the type is only known at runtime, effectively disabling static checks for that type. More interestingly, we can also place upper and lower bounds on the dynamic type using set operations. As we will soon learn, this will reveal interesting properties about our type system. + +It is often said that gradual typing is the best of both words. Perhaps ironically, that's true and false at the same time. If you use a gradual type system but you never use the `dynamic()` type, then it behaves exactly like a static type system. However, the more you use the `dynamic()` type, the fewer guarantees the type system will give you, the more the `dynamic()` type propagates through the system. Therefore, it is in our interest to reduce the occurrences of the `dynamic()` type as much as possible, and that's what we set out to do. + +## Interfacing static and dynamic code: the trouble with `dynamic()` + +Let's go back to our constrained identity function that accepts only numbers: + +```elixir +$ a -> a when a: number() +def identity(arg), do: arg +``` + +Now imagine that we have some untyped code that calls this function: + +```elixir +def debug(arg) do + "we got: " <> identity(arg) +end +``` + +Since `debug/1` is untyped, its argument will receive the type `dynamic()`. + +`debug/1` proceeds to call `identity` with an argument and then uses the string concatenation operator (`<>`) to concatenate `"we got: "` to the result of `identity(arg)`. Since `identity/1` is meant to return a number and string concatenation requires two strings as operands, there is a typing error in this program. On the other hand, if you call `debug("hello")` at runtime, the code will work and won't raise any exceptions. + +In other words, the static typing version of the program and its runtime execution do not match in behaviour. So how do we tackle this? + +One option is to say that's all behaving as expected. If `debug/1` is untyped, its `arg` has the `dynamic()` type. To type check this program, we specify that `identity(dynamic())` returns the `dynamic()` type, the concatenation of a string with `dynamic()` also returns `dynamic()`, and consequently `debug/1` gets the type `dynamic() -> dynamic()`, with no type errors emitted. + +The trouble is: this is not a very useful choice. Once `dynamic()` enters the system, it _spreads everywhere_, we perform fewer checks, effectively discarding the information that `identity/1` returns a number, and the overall type system becomes less useful. + +Another option would be for us to say: once we call a statically typed function with `dynamic()`, we will ignore the `dynamic()` type. If the function says it returns a `number()`, then it will surely be a number! In this version, `identity(dynamic())` returns `number()` and the type system will catch a type error when concatenating a string with a number. + +This is similar to the approach taken by TypeScript. This means we can perform further static checks, but it also means we can call `debug("foobar")` and that will return the string `"we got: foobar"`! But how can that be possible when the type system told us that `identity` returns a `number()`? This can lead to confusion and surprising results at runtime. We say this system is unsound, because the types at runtime do not match our compile-time types. + +None of our solutions so far attempted to match the static and runtime behaviors, but rather, they picked one in favor of the other. + +But don't despair, there is yet another option. We could introduce runtime checks whenever we cross the "dynamic <-> static" boundaries. In this case, we could say `identity(dynamic())` returns a `number()`, but we will introduce a runtime check into the code to guarantee that's the case. This means we get static checks, we ensure the value is correct at runtime, with the cost of introducing additional checks at runtime. Unfortunately, those checks may affect performance, depending on the complexity of the data structure and on how many times we cross the "dynamic <-> static" boundary. + +> Note: there is [recent research in using the runtime checks introduced by a gradual type system to provide compiler optimizations](https://arxiv.org/abs/2206.13831). Some of these techniques are already leveraged by the Erlang VM to optimize code based on patterns and guards. + +To summarize, we have three options: + + * Calling static code from dynamic code returns `dynamic()`, dropping the opportunity of further static typing checks (this is sound) + + * Calling static code from dynamic code returns the static types, potentially leading to mismatched types at runtime (this is unsound) + + * Calling static code from dynamic code returns the static types with additional runtime checks, unifying both behaviours but potentially impacting performance (this is sound) + +## Introducing strong arrows + +I have always said that Elixir, thanks to Erlang, is an assertive language. For example, if our identity function is restricted to only numbers, in practice we would most likely write it as: + +```elixir +$ a -> a when a: number() +def identity(arg) when is_number(arg), do: arg +``` + +In the example above, `identity` will fail if given any value that is not a number. We often rely on pattern matching and guards and, in turn, they helps us assert on the types we are working with. Not only that, Erlang's JIT compiler already relies on this information to [perform optimizations](https://www.erlang.org/blog/type-based-optimizations-in-the-jit/) whenever possible. + +We also say Elixir is strongly typed because its functions and operators avoid implicit type conversions. The following functions also fail when their input does not match their type: + +```elixir +$ binary() -> binary() +def debug(string), do: "we got: " <> string + +$ (integer() -> integer()) and (float() -> float()) +def increment(number), do: number + 1 +``` + +`<>` only accepts binaries as arguments and will raise otherwise. `+` only accepts numbers (integers or floats) and will raise otherwise. `+` does not perform implicit conversions of non-numeric types, such as strings to number, as we can see next: + +```elixir +iex(1)> increment(1) +2 +iex(2)> increment(13.0) +14.0 +iex(3)> increment("foobar") +** (ArithmeticError) bad argument in arithmetic expression: "foobar" + 1 +``` + +In other words, Elixir's runtime consistently checks the values and their types at runtime. If `increment` fails when given something else than a number, then it will fail when the `dynamic()` type does not match its input at runtime. This guarantees `increment` returns its declared type and therefore we do not need to introduce runtime type checks when calling said function from untyped code. + +When we look at the `identity`, `debug`, and `increment` functions above, we - as developers - can state that these functions raise when given a value that does not match their input. However, how can we generalize this property so it is computed by the type system itself? To do so, we introduce a new concept called **strong arrows**, which relies on set-theoretical types to derive this property. + +The idea goes as follows: a strong arrow is a function that can be statically proven that, when evaluated on values outside of its input types (i.e. its domain), it will error. For example, in our `increment` function, if we pass a `string()` as argument, it won't type check, because `string() + integer()` is not a valid operation. Thanks to set-theoretic types, we can compute all values outside of the domain by computing the negation of a set. Given `increment/1` will fail for all types which are `not number()`, the function is strong. + +By applying this rule to all typed functions, we will know which functions are strong and which ones are not. If a function is strong, the type system knows that calling it with a `dynamic()` type will always evaluate to its return type! Therefore we say the return type of `increment(dynamic())` is `number()`, which is sound and does not need further runtime checks! + +Going back to our `debug` function, when used with a guarded identity, it will be able to emit warnings at compile-time, errors at runtime, without introducing any additional runtime check: + +```elixir +$ a -> a when a: number() +def identity(arg) when is_number(arg), do: arg + +def debug(arg) do + "we got: " <> identity(arg) +end +``` + +However, if the `identity` function is not strong, then we must fallback to one of the strategies in the previous section. + +Another powerful property of strong arrows is that they are composable. Let's pick an example from the paper: + +```elixir +$ number(), number() -> number() +def subtract(a, b) do + a + negate(b) +end + +$ number() -> number() +def negate(int), do: -int +``` + +In the example above, `negate/1`'s type is a strong arrow, as it raises for any input outside of its domain. `subtract/2`'s type is also a strong arrow, because both `+` and our own `negate` are strong arrows too. This is an important capability as it limits how `dynamic()` types spread throughout the system. + +> Errata: my presentation used the type `integer()` instead of `number()` for the example above. However, that was a mistake in the slide. Giving the type `integer(), integer() -> integer()` to `subtract` and `integer() -> integer()` to `negate` does not make `subtract` a strong arrow. Can you tell why? + +Luckily, other gradually typed languages can also leverage strong arrows. However, the more polymorphic a language and its functions are, the more unlikely it is to conclude that a given function is strong. For example, in other gradually typed languages such as Python or Ruby, the `+` operator is extensible and the user can define custom types where the operation is valid. In TypeScript, `"foobar" + 1` is also a valid operation, which expands the function domain. In both cases, an `increment` function restricted to numbers would not have a strong arrow type, as the operator won't fail for all types outside of `number()`. Therefore, to remain sound, they must either restrict the operands with further runtime checks or return `dynamic()` (reducing the number of compile-time checks). + +There is one last scenario to consider, which I did not include during my keynote for brevity. Take this function: + +```elixir +$ integer() -> :ok +def receives_integer_and_returns_ok(_arg), do: :ok +``` + +The function above can receive any type and return `:ok`. Is its type a strong arrow? Well, according to our definition, it is not. If we negate its input, type checking does not fail, it returns `:ok`. + +However, given the return type is always the same, it should be a strong arrow! To do so, let's amend and rephrase our definition of strong arrows: we negate the domain (i.e. the inputs) of a function and then type check it. If the function returns `none()` (i.e. it does not type check) or a type which is a subset of its codomain (i.e. its output), then it is a strong arrow. + +## Gradual typing and false positives + +There is one last scenario we must take into consideration when interfacing dynamic and static code. Imagine the following code: + +```elixir +def increment_and_remainder(numerator, denominator) do + rem(numerator, increment(denominator)) +end + +$ (integer() -> integer()) and (float() -> float()) +def increment(number), do: number + 1 +``` + +The `increment_and_remainder/2` function is untyped, therefore both of its arguments receive type `dynamic()`. The function then computes the remainder of the numerator by the denominator incremented by one. For this example, let's assume all uses of `increment_and_remainder/2` in our program passes two integers as arguments. + +Given `increment/1` has a strong arrow type, according to our definition, `increment(dynamic())` will return `integer() or float()` (also known as `number()`). Here lies the issue: if `increment(dynamic())` returns `integer() or float()`, the program above won't type check because `rem/2` does not accept floats. + +When faced with this problem, there are two possible reactions: + +1. It is correct for the function to not type check given `increment` may return a float + +2. It is incorrect for the function to not type check because the error it describes never occurs in the codebase + +Another interesting property of gradual set-theoretic types is that we can also place upper bounds on the `dynamic()` type. If a function returns `number()`, it means the caller needs to handle both `integer()` and `float()`. However, if a function returns `dynamic() and number()`, it means the type is defined at runtime, but it must still verify it is one of `integer()` or `float()` at compile time. + +Therefore, `rem/2` will type check if its second argument has the type `dynamic() and number()`, as there is one type at runtime (`integer()`) that satisfies type checking. On the other hand, if you attempt to use the string concatenation operator (`<>`) on `dynamic() and number()`, then there is no acceptable runtime type and you'd still get a typing violation! + +Going back to strong arrows, there are two possible return types from a strong arrow: + +1. A strong arrow, when presented with a dynamic type, returns its codomain + +2. A strong arrow, when presented with a dynamic type, returns the intersection of the codomain with the `dynamic()` type + +The second option opens up the possibility for existing codebases to gradually migrate to static types without dealing with false positives. Coming from a dynamic background, false positives can be seen as noisy or as an indication that static types are not worth the trouble. With strong arrows and gradual set-theoretic types, we will be able to explore different trade-offs on mixed codebases. Which of the two choices above we will adopt as a default and how to customize them is yet to be decided. It will depend on the community feedback as we experiment and integrate the type system. + +Erlang and Elixir developers who use Dialyzer will be familiar with these trade-offs, as the second option mirrors Dialyzer's behaviour of no false positives. The difference here is that our semantics are integrated into a complete type system. If no type signature is present, the `dynamic()` type is used, and we will leverage the techniques described here to interface dynamic and static code. If a function has a type signature, and no `dynamic()` type is present, then it will behave as statically typed code when called with statically typed arguments. Migrating to static types will naturally reduce the interaction points between dynamic and static code, removing the reliance on the `dynamic()` type. + +## Summary + +Set-theoretic types allow us to express many typing features based on set operations of union, intersection, and negation. + +In particular, we have been exploring a gradual set-theoretic type system for Elixir, paying special attention to how the type system will integrate with existing codebases and how it can best leverage the semantics of the Erlang Virtual Machine. The type system will also perform limited inference based on patterns and guards (as described in the paper), which - in addition to strong arrows - we hope to bring some of the benefits of static typing to codebases without changing a single line of code. + +While our efforts have officially moved from research into development, and [we have outlined an implementation plan](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/), we haven't yet fully implemented nor assessed the usability of set-theoretic types in existing Elixir codebases, either large or small. There is much to implement and validate, and we don't rule the possibility of finding unforeseen deal breakers that could send us back to square one. Yet we are pleased and cautiously excited with the new developments so far. + +The development of Elixir's type system is sponsored by [Fresha](https://www.fresha.com) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team) ([they are hiring!](https://starfish.team/jobs/experienced-elixir-developer)), +and [Dashbit](https://dashbit.co). diff --git a/_posts/2023-12-22-elixir-v1-16-0-released.markdown b/_posts/2023-12-22-elixir-v1-16-0-released.markdown new file mode 100644 index 000000000..6e4a61dca --- /dev/null +++ b/_posts/2023-12-22-elixir-v1-16-0-released.markdown @@ -0,0 +1,189 @@ +--- +layout: post +title: Elixir v1.16 released +authors: +- José Valim +category: Releases +excerpt: Elixir v1.16 released with compiler diagnostics and extensive documentation +--- + +Elixir v1.16 has just been released. 🎉 + +The Elixir team continues improving the developer experience +via tooling, documentation, and precise feedback, while keeping +the language stable and compatible. + +The notable improvements in this release are the addition of +compiler diagnostics and extensive improvements to our docs +in the forms of guides, anti-patterns, diagrams and more. + +## Code snippets in diagnostics + +Elixir v1.15 introduced a new compiler diagnostic format and +the ability to print multiple error diagnostics per compilation +(in addition to multiple warnings). + +With Elixir v1.16, we also include code snippets in exceptions +and diagnostics raised by the compiler, including ANSI coloring +on supported terminals. For example, a syntax error now includes +a pointer to where the error happened: + +``` +** (SyntaxError) invalid syntax found on lib/my_app.ex:1:17: + error: syntax error before: '*' + │ + 1 │ [1, 2, 3, 4, 5, *] + │ ^ + │ + └─ lib/my_app.ex:1:17 +``` + +For mismatched delimiters, it now shows both delimiters: + +``` +** (MismatchedDelimiterError) mismatched delimiter found on lib/my_app.ex:1:18: + error: unexpected token: ) + │ + 1 │ [1, 2, 3, 4, 5, 6) + │ │ └ mismatched closing delimiter (expected "]") + │ └ unclosed delimiter + │ + └─ lib/my_app.ex:1:18 +``` + +For unclosed delimiters, it now shows where the unclosed delimiter starts: + +``` +** (TokenMissingError) token missing on lib/my_app:8:23: + error: missing terminator: ) + │ + 1 │ my_numbers = (1, 2, 3, 4, 5, 6 + │ └ unclosed delimiter + ... + 8 │ IO.inspect(my_numbers) + │ └ missing closing delimiter (expected ")") + │ + └─ lib/my_app:8:23 +``` + +Errors and warnings diagnostics also include code snippets. +When possible, we will show precise spans, such as on undefined variables: + +``` + error: undefined variable "unknown_var" + │ +5 │ a - unknown_var + │ ^^^^^^^^^^^ + │ + └─ lib/sample.ex:5:9: Sample.foo/1 +``` + +Otherwise the whole line is underlined: + +``` +error: function names should start with lowercase characters or underscore, invalid name CamelCase + │ +3 │ def CamelCase do + │ ^^^^^^^^^^^^^^^^ + │ + └─ lib/sample.ex:3 +``` + +A huge thank you to Vinícius Müller for working on the new diagnostics. + +## Revamped documentation + +The [ExDoc](https://github.com/elixir-lang/ex_doc) package provides Elixir developers +with one of the most complete and robust documentation generator. It [supports API +references, tutorials, cheatsheets, and more](/blog/2022/12/22/cheatsheets-and-8-other-features-in-exdoc-that-improve-the-developer-experience/). + +However, because many of the language tutorials and reference documentation +were written before ExDoc, they were maintained separately as part of the +official website, separate from the language source code. With Elixir v1.16, +[we have moved our learning material to the language repository](https://hexdocs.pm/elixir/introduction.html). +This provides several benefits: + +1. Tutorials are versioned alongside their relevant Elixir version + +2. You get full-text search across all API reference and tutorials + +3. ExDoc will autolink module and function names in tutorials to their relevant API documentation + +Another feature we have incorporated in this release is the addition +of cheatsheets, starting with [a cheatsheet for the Enum module](https://hexdocs.pm/elixir/main/enum-cheat.html). +If you would like to contribute future cheatsheets to Elixir itself, +feel free to start a discussion and collect feedback on the +[Elixir Forum](https://elixirforum.com/). + +Finally, we have started enriching our documentation with +[Mermaid.js](https://mermaid.js.org/) diagrams. You can find examples +in the [GenServer](https://hexdocs.pm/elixir/GenServer.html) +and [Supervisor](https://hexdocs.pm/elixir/Supervisor.html) docs. + +Elixir has always been praised by its excellent documentation and +we are glad to continue to raise the bar for the whole ecosystem. + +## Living anti-patterns reference + +Elixir v1.16 incorporates and extends the work on [Understanding Code Smells +in Elixir Functional Language](https://github.com/lucasvegi/Elixir-Code-Smells/blob/main/etc/2023-emse-code-smells-elixir.pdf), +by Lucas Vegi and Marco Tulio Valente, from [ASERG/DCC/UFMG](http://aserg.labsoft.dcc.ufmg.br/), +into [the official documention in the form of anti-patterns](https://hexdocs.pm/elixir/what-anti-patterns.html). +Our goal is to provide examples of potential pitfalls for library and +application developers, with additional context and guidance on how +to improve their codebases. + +In earlier versions, Elixir's official reference for library authors +included a list of anti-patterns for library developers. Lucas Vegi and +Marco Tulio Valente extended and refined this list based on the existing +literature, articles, and community input (including feedback based on +their prevalence in actual codebases). + +To incorporate the anti-patterns into the language, we trimmed the list down +to keep only anti-patterns which are unambiguous and actionable, and divided +them into four categories: [code-related](https://hexdocs.pm/elixir/code-anti-patterns.html), +[design-related](https://hexdocs.pm/elixir/design-anti-patterns.html), +[process-related](https://hexdocs.pm/elixir/process-anti-patterns.html), +and [meta-programming](https://hexdocs.pm/elixir/macro-anti-patterns.html). +Then we collected more community feedback during the release candidate +period, further refining and removing unclear guidance. + +We are quite happy with the current iteration of anti-patterns but +this is just the beginning. As they become available to the whole community, +we expect to receive more input, questions, and concerns. We will +continue listening and improving, as our ultimate goal is to provide +a live reference that reflects the practices of the ecosystem, +rather than a document that is written in stone and ultimately gets +out of date. A perfect example of this is [the recent addition of +"Sending unnecessary data" anti-pattern](https://github.com/elixir-lang/elixir/pull/13194), +which was contributed by the community and describes a pitfall that may +happen across codebases. + +## Type system updates + +As we get Elixir v1.16 out of door, the Elixir team will focus on bringing +the initial core for set-theoretic types into the Elixir compiler, with the +goal of running automated analysis in patterns and guards. This is [the first +step outlined in a previous article](/blog/2023/06/22/type-system-updates-research-dev/) +and is sponsored by [Fresha](https://www.fresha.com) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team) ([they are hiring!](https://starfish.team/jobs/experienced-elixir-developer)), +and [Dashbit](https://dashbit.co). + +## Learn more + +Other notable changes in this release are: + +* the addition of [`String.replace_invalid/2`](https://hexdocs.pm/elixir/String.html#replace_invalid/2), to help deal with invalid UTF-8 encoding + +* the addition of the `:limit` option in [`Task.yield_many/2`](https://hexdocs.pm/elixir/Task.html#yield_many/2) that limits the maximum number of tasks to yield + +* improved binary pattern matching by allowing prefix binary matches, such as `<<^prefix::binary, rest::binary>>` + +For a complete list of all changes, see the +[full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.16.0). + +Check [the Install section](/install.html) to get Elixir installed and +read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +to learn more. + +Happy learning! diff --git a/_posts/2024-03-05-veeps-elixir-case.markdown b/_posts/2024-03-05-veeps-elixir-case.markdown new file mode 100644 index 000000000..f0e60b7d1 --- /dev/null +++ b/_posts/2024-03-05-veeps-elixir-case.markdown @@ -0,0 +1,135 @@ +--- +layout: post +title: "Scaling a streaming service to hundreds of thousands of concurrent viewers at Veeps" +authors: +- Hugo Baraúna +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Veeps. +logo: /images/cases/logos/veeps.svg +tags: streaming scaling web +--- + +*Welcome to our series of [case studies about companies using Elixir in production](/cases.html).* + +[Veeps](https://veeps.com/) is a streaming service that offers direct access to live and on-demand events by award-winning artists at the most iconic venues. Founded in 2018, it became part of Live Nation Entertainment in 2021. + +Veeps have been named [one of the ten most innovative companies in music](https://www.fastcompany.com/90848907/most-innovative-companies-music-2023) and nominated for an Emmy. They currently hold the [Guinness World Record](https://www.guinnessworldrecords.com/world-records/650975-most-tickets-sold-for-a-livestreamed-concert-by-a-solo-male-artist-current-year) for the world's largest ticketed livestream by a solo male artist—a performance where Elixir and Phoenix played an important role in the backend during the streaming. + +This case study examines how Elixir drove Veeps' technical transformation, surpassing high-scale demands while keeping the development team engaged and productive. + + +## The challenge: scaling to hundreds of thousands of simultaneous users + +Imagine you are tasked with building a system that can livestream a music concert to hundreds of thousands of viewers around the world at the same time. + +In some cases, users must purchase a ticket before the concert can be accessed. For a famous artist, it’s not uncommon to see thousands of fans continuously refreshing their browsers and attempting to buy tickets within the first few minutes of the announcement. + +The Veeps engineering team needed to handle both challenges. + +Early on, the Veeps backend was implemented in [Ruby on Rails](https://rubyonrails.org/). Its first version could handle a few thousand simultaneous users watching a concert without any impact to stream quality, which was fine when you have a handful of shows but would be insufficient with the expected show load and massive increase in concurrent viewership across streams. It was around that time that [Vincent Franco](https://twitter.com/vinniefranco) joined Veeps as their CTO. + +Vincent had an extensive background in building and maintaining ticketing and event management software at scale. So, he used that experience to further improve the system to handle tens of thousands of concurrent users. However, it became clear that improving it to *hundreds* of thousands would be a difficult challenge, requiring substantial engineering efforts and increased operational costs. The team began evaluating other stacks that could provide the out-of-the-box tooling for scaling in order to reach both short and long-term goals. + + +## Adopting Elixir, hiring, and rewriting the system + +Vincent, who had successfully deployed Elixir as part of high-volume systems in the past, believed Elixir was an excellent fit for Veeps' requirements. + +Backed by his experience and several case studies from the Elixir community, such as [the one from Discord](/blog/2020/10/08/real-time-communication-at-scale-with-elixir-at-discord/), Vincent convinced management that Elixir could address their immediate scaling needs and become a reliable foundation on which the company could build. + +With buy-in from management, the plan was set in motion. They had two outstanding goals: + +* Prepare the platform to welcome the most famous artists in the world. +* Build their own team of engineers to help innovate and evolve the product. + +Vincent knew that hiring right-fit technical people can take time and he didn't want to rush the process. Hence, he hired [DockYard](https://dockyard.com/) to rebuild the system while simultaneously searching for the right candidates to build out the team. + +Eight months later, the system had been entirely rewritten in Elixir and Phoenix. Phoenix Channels were used to enrich the live concert experience, while Phoenix LiveView empowered the ticket shopping journey. + +The rewrite was put to the test shortly after with a livestream that remains one of Veeps’ biggest, still to this day. Before the rewrite, 20 Rails nodes were used during big events, whereas now, the same service requires only 2 Elixir nodes. And the new platform was able to handle 83x more concurrent users than the previous system. + +The increase in infrastructure efficiency significantly reduced the need for complex auto-scaling solutions while providing ample capacity to handle high-traffic spikes. + +> The rewrite marked the most extensive and intricate system migration in my career, and yet, it was also the smoothest. +> +> \- Vincent Franco, CTO + + +This was a big testament to Elixir and Phoenix's scalability and gave the team confidence that they made the right choice. + +By the time the migration was completed, Veeps had also assembled an incredibly talented team of two backend and two frontend engineers, which continued to expand and grow the product. + + +## Perceived benefits of using Elixir and its ecosystem + +After using Elixir for more than two years, Veeps has experienced significant benefits. Here are a few of them. + + +### Architectural simplicity + +Different parts of the Veeps system have different scalability requirements. For instance, when streaming a show, the backend receives metadata from users' devices every 30 seconds to track viewership. This is the so-called *Beaconing service*. + +Say you have 250,000 people watching a concert: the Beaconing service needs to handle thousands of requests per second for a few hours at a time. As a result, it needs to scale differently from other parts of the system, such as the merchandise e-commerce or backstage management. + +To tackle this issue, they built a distributed system. They packaged each subsystem as an [Elixir release](https://hexdocs.pm/elixir/config-and-releases.html#releases), totaling five releases. For the communication layer, they used distributed Erlang, which is built into Erlang/OTP, allowing seamless inter-process communication across networked nodes. + +In a nutshell, each node contains several processes with specific responsibilities. Each of these processes belongs to their respective [distributed process group](https://www.erlang.org/doc/man/pg.html). If node A needs billing information, it will reach out to any process within the "billing process group", which may be anywhere in the cluster. + +When deploying a new version of the system, they deploy a new cluster altogether, with all five subsystems at once. Given Elixir's scalability, the whole system uses 9 nodes, making a simple deployment strategy affordable and practical. As we will see, this approach is well-supported during development too, thanks to the use of Umbrella Projects. + + +### Service-oriented architecture within a monorepo + +Although they run a distributed system, they organize the code in only one repository, following the monorepo approach. To do that, they use the [Umbrella Project feature](https://hexdocs.pm/elixir/dependencies-and-umbrella-projects.html#content) from Mix, the build tool that ships with Elixir. + +Their umbrella project consists of 16 applications (at the time of writing), which they [sliced into five OTP releases](https://hexdocs.pm/mix/Mix.Tasks.Release.html#module-umbrellas). The remaining applications contain code that needs to be shared between multiple applications. For example, one of the shared applications defines all the structs sent as messages across the subsystems, guaranteeing that all subsystems use the same schemas for that exchanged data. + +> With umbrella projects, you can have the developer experience benefits of a single code repository, while being able to build a service-oriented architecture. +> +> \- Andrea Leopardi, Principal Engineer + + +### Reducing complexity with the Erlang/Elixir toolbox + +Veeps has an e-commerce platform that allows concert viewers to purchase artist merchandise. In e-commerce, a common concept is a shopping cart. Veeps associates each shopping cart as a [GenServer](https://hexdocs.pm/elixir/GenServer.html), which is a lightweight process managed by the Erlang VM. + +This decision made it easier for them to implement other business requirements, such as locking the cart during payments and shopping cart expiration. Since each cart is a process, the expiration is as simple as sending a message to a cart process based on a timer, which is easy to do using GenServers. + +For caching, the team relies on [ETS (Erlang Term Storage)](https://www.erlang.org/doc/man/ets.html), a high-performing key-value store part of the Erlang standard library. For cache busting between multiple parts of the distributed system, they use [Phoenix PubSub](https://github.com/phoenixframework/phoenix_pubsub), a real-time publisher/subscriber library that comes with [Phoenix](https://phoenixframework.org/). + +Before the rewrite, the Beaconing service used Google's Firebase. Now, the system uses [Broadway](https://elixir-broadway.org/) to ingest data from hundreds of thousands of HTTP requests from concurrent users. Broadway is an Elixir library for building concurrent data ingestion and processing pipelines. They utilized the library's capabilities to efficiently send requests to AWS services, regulating batch sizes to comply with AWS limits. They also used it to handle rate limiting to adhere to AWS service constraints. All of this was achieved with Broadway's built-in functionality. + +Finally, they use [Oban](https://getoban.pro/), an Elixir library for background jobs, for all sorts of background-work use cases. + +Throughout the development journey, Veeps consistently found that Elixir and its ecosystem had built-in solutions for their technical challenges. Here's what Vincent, CTO of Veeps, had to say about that: + +> Throughout my career, I've worked with large-scale systems at several companies. However, at Veeps, it's unique because we achieve this scale with minimal reliance on external tools. It's primarily just Elixir and its ecosystem that empower us. +> +> \- Vincent Franco, CTO + +This operational simplicity benefitted not only the production environment but also the development side. The team could focus on learning Elixir and its ecosystem without the need to master additional technologies, resulting in increased productivity. + + +### LiveView: simplifying the interaction between front-end and back-end developers + +After the rewrite, [LiveView](https://github.com/phoenixframework/phoenix_live_view), a Phoenix library for building interactive, real-time web apps, was used for every part of the front-end except for the "Onstage" subsystem (responsible for the live stream itself). + +The two front-end developers, who came from a React background, also started writing LiveView. After this new experience, the team found the process of API negotiation between the front-end and back-end engineers much simpler compared to when using React. This was because they only had to use Elixir modules and functions instead of creating additional HTTP API endpoints and all the extra work that comes with them, such as API versioning. + +> Our front-end team, originally proficient in React, has made a remarkable transition to LiveView. They've wholeheartedly embraced its user-friendly nature and its smooth integration into our system. +> +> \- Vincent Franco, CTO + + +## Conclusion: insights from Veeps' Elixir experience + +The decision to use Elixir has paid dividends beyond just system scalability. The team, with varied backgrounds in Java, PHP, Ruby, Python, and Javascript, found Elixir's ecosystem to be a harmonious balance of simplicity and power. + +By embracing Elixir's ecosystem, including Erlang/OTP, Phoenix, LiveView, and Broadway, they built a robust system, eliminated the need for numerous external dependencies, and kept productively developing new features. + + +> Throughout my career, I've never encountered a developer experience as exceptional as this. Whether it's about quantity or complexity, tasks seem to flow effortlessly. The team's morale is soaring, everyone is satisfied, and there's an unmistakable atmosphere of positivity. We're all unequivocally enthusiastic about this language. +> +> \- Vincent Franco, CTO + +Veeps' case illustrates how Elixir effectively handles high-scale challenges while keeping the development process straightforward and developer-friendly. diff --git a/_posts/2024-06-12-elixir-v1-17-0-released.markdown b/_posts/2024-06-12-elixir-v1-17-0-released.markdown new file mode 100644 index 000000000..c857c906e --- /dev/null +++ b/_posts/2024-06-12-elixir-v1-17-0-released.markdown @@ -0,0 +1,178 @@ +--- +layout: post +title: "Elixir v1.17 released: set-theoretic data types, calendar durations, and Erlang/OTP 27 support" +authors: +- Andrea Leopardi +category: Releases +excerpt: "Elixir v1.17 released: set-theoretic data types, calendar durations, and Erlang/OTP 27 support" +--- + +Elixir v1.17 has just been released. 🎉 + +This release introduces set-theoretic types into a handful of language constructs. While there are still [many steps ahead of us](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/), this important milestone already brings benefits to developers in the form of new warnings for common mistakes. This new version also adds support for [Erlang/OTP 27](https://www.erlang.org/downloads/27), the latest and greatest Erlang release. You'll also find a new calendar-related data type (`Duration`) and a `Date.shift/2` function. + +Let's dive in. + +## Warnings from gradual set-theoretic types + +This release introduces gradual set-theoretic types to infer types from patterns and use them to type check programs, enabling the Elixir compiler to find faults and bugs in codebases without requiring changes to existing software. The underlying principles, theory, and roadmap of our work have been outlined in ["The Design Principles of the Elixir Type System" by Giuseppe Castagna, Guillaume Duboc, José Valim](https://arxiv.org/abs/2306.06391). + +At the moment, Elixir developers will interact with set-theoretic types only through **warnings** found by the type system. The current implementation models all data types in the language: + + * `binary()`, `integer()`, `float()`, `pid()`, `port()`, `reference()` - these + types are indivisible. This means both `1` and `13` get the same `integer()` + type. + + * `atom()` - it represents all atoms and it is divisible. For instance, the + atom `:foo` and `:hello_world` are also valid (distinct) types. + + * `map()` and structs - maps can be "closed" or "open". Closed maps only allow + the specified keys, such as `%{key: atom(), value: integer()}`. Open maps + support any other keys in addition to the ones listed and their definition + starts with `...`, such as `%{..., key: atom(), value: integer()}`. Structs + are closed maps with the `__struct__` key. + + * `tuple()`, `list()`, and `function()` - currently they are modelled as + indivisible types. The next Elixir versions will also introduce fine-grained + support to them. + +We focused on *atoms* and *maps* on this initial release as they are respectively the simplest and the most complex types representations, so we can stress the performance of the type system and quality of error messages. Modelling these types will also provide the most immediate benefits to Elixir developers. Assuming there is a variable named `user`, holding a `%User{}` struct with a `address` field, Elixir v1.17 will emit the following warnings at compile-time: + + * Pattern matching against a map or a struct that does not have the given key, + such as `%{adress: ...} = user` (notice `address` vs `adress`). + + * Accessing a key on a map or a struct that does not have the given key, such + as `user.adress`. + + * Invoking a function on non-modules, such as `user.address()`. + + * Capturing a function on non-modules, such as `&user.address/0`. + + * Attempting to call an anonymous function without an actual function, such as + `user.()`. + + * Performing structural comparisons between structs, such as `my_date < + ~D[2010-04-17]`. + + * Performing structural comparisons between non-overlapping types, such as + `integer >= string`. + + * Building and pattern matching on binaries without the relevant specifiers, + such as `<>` (this warns because by default it expects an integer, it + should have been `<>` instead). + + * Attempting to rescue an undefined exception or a struct that is not an + exception. + + * Accessing a field that is not defined in a rescued exception. + +Here's an example of how the warning for accessing a misspelled field of a +struct looks like: + +![Example of a warning when accessing a mispelled struct field](/images/contents/type-warning-on-struct-field.png) + +Another example, this time it's a warning for structural comparison across two +`Date` structs: + +![Example of a warning when comparing two structs with ">"](/images/contents/type-warning-on-date-comparison.png) + +These warnings also work natively in text editors, as they are standard Elixir +compiler warnings: + +![Example of a type warning inline in an editor](/images/contents/type-warning-in-editor.png) + +These new warnings will help Elixir developers find bugs earlier and give more +confidence when refactoring code, especially around maps and structs. While +Elixir already emitted some of these warnings in the past, those were discovered +using syntax analysis. The new warnings are more reliable, precise, and with +better error messages. Keep in mind, however, that the Elixir typechecker only +infers types from patterns within the same function at the moment. Analysis from +guards and across function boundaries will be added in future releases. For more +details, see our new [reference document on gradual set-theoretic +types](https://hexdocs.pm/elixir/gradual-set-theoretic-types.html). + +The type system was made possible thanks to a partnership between +[CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development +work is currently sponsored by [Fresha](https://www.fresha.com/) +([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), +[Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +## Erlang/OTP support + +This release adds support for Erlang/OTP 27 and drops support for Erlang/OTP 24. +We recommend Elixir developers to migrate to Erlang/OTP 26 or later, especially +on Windows. Support for WERL (a graphical user interface for the Erlang terminal +on Windows) will be removed in Elixir v1.18. + +You can read more about Erlang/OTP 27 in [their release +announcement](https://www.erlang.org/downloads/27). The bits that are +particularly interesting for Elixir developers are the addition of a [`json` +module](https://erlang.org/documentation/doc-15.0-rc3/lib/stdlib-6.0/doc/html/json.html) +and process labels (`proc_lib:set_label/1`). The latter will also be available +in this Elixir release as `Process.set_label/1`. + +## New `Duration` data type and shifting functions + +This Elixir version introduces the `Duration` data type and APIs to shift dates, +times, and date times by a given duration, considering different calendars and +time zones. + +```elixir +iex> Date.shift(~D[2016-01-31], month: 2) +~D[2016-03-31] +``` + +We chose the name *"shift"* for this operation (instead of "add") since working +with durations does not obey properties such as **associativity**. For instance, +adding one month and then one month does not give the same result as adding two +months: + +```elixir +iex> ~D[2016-01-31] |> Date.shift(month: 1) |> Date.shift(month: 1) +~D[2016-03-29] +``` + +Still, durations are essential for building intervals, recurring events, and +modelling scheduling complexities found in the world around us. For `DateTime`s, +Elixir will correctly deal with time zone changes (such as Daylight Saving +Time). However, provisions are also available in case you want to surface +conflicts, such as shifting to a wall clock that does not exist, because the +clock has been moved forward by one hour. See `DateTime.shift/2` for examples. + +Finally, we added a new `Kernel.to_timeout/1` function, which helps developers +normalize durations and integers to a timeout used by many APIs—like `Process`, +`GenServer`, and more. For example, to send a message after one hour, you can +now write: + +```elixir +Process.send_after(pid, :wake_up, to_timeout(hour: 1)) +``` + +## Learn more + +Here are other notable changes in this release: + + * There are new `Keyword.intersect/2,3` functions to mirror the equivalent in + the `Map` module. + + * A new Mix profiler was added, `mix profile.tprof`, which lets you use the + new [tprof](https://www.erlang.org/doc/apps/tools/tprof.html) + profiler released with Erlang/OTP 27. This profiler leads to the + soft-deprecation of `mix profile.cprof` and `mix profile.eprof`. + + * We added `Kernel.is_non_struct_map/1`, a new guard to help with the common + pitfall of matching on `%{}`, which also successfully matches structs (as + they are maps underneath). + + * Elixir's Logger now formats + [`gen_statem`](https://www.erlang.org/doc/apps/stdlib/gen_statem.html) + reports and includes Erlang/OTP 27 *process labels* in logger events. + +For a complete list of all changes, see the +[full release notes](https://github.com/elixir-lang/elixir/releases/tag/v1.17.0). + +Check [the Install section](/install.html) to get Elixir installed and +read our [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +to learn more. + +Happy learning! diff --git a/_posts/2024-08-15-welcome-elixir-language-server-team.markdown b/_posts/2024-08-15-welcome-elixir-language-server-team.markdown new file mode 100644 index 000000000..b9e29ca0f --- /dev/null +++ b/_posts/2024-08-15-welcome-elixir-language-server-team.markdown @@ -0,0 +1,91 @@ +--- +layout: post +title: "Announcing the official Elixir Language Server team" +authors: +- José Valim +category: Announcements +excerpt: "Announcing the official Elixir Language Server team to work on the code intelligence infrastructure to be used across tools and editors" +--- + +> Update: [the new language server is called Expert and is currently available in alpha as open source](https://github.com/elixir-lang/expert). Companies wanting to directly sponsor work on Expert, please reach out to [Dan Janowski](mailto:sponsor+expert@erlef.org) from the [Erlang Ecosystem Foundation](https://erlef.org), which has kindly stepped in to manage and bring visibility into the project sponsorships. + +I am glad to welcome Elixir's official Language Server team, formed by (in alphabetical order): + +* [Jonatan Kłosko](https://github.com/jonatanklosko) +* [Łukasz Samson](https://github.com/lukaszsamson) +* [Mitch Hanberg](https://www.mitchellhanberg.com/) +* [Steve Cohen](https://github.com/scohen) + +The team will work on the code intelligence infrastructure to be used across tools and editors. These efforts are partially funded by [Fly.io](https://fly.io/) and [Livebook](https://livebook.dev/). + +## A brief history + +The [Language Server Protocol (LSP)](https://en.wikipedia.org/wiki/Language_Server_Protocol) was created by Microsoft as a protocol between IDEs and programming languages to provide language intelligence tools. + +The first implementation of said protocol for Elixir was started by [Jake Becker](https://github.com/JakeBecker/elixir-ls/), back in 2017, alongside an implementation for Visual Studio Code, and it relied on [the ElixirSense project from Marlus Saraiva](https://github.com/msaraiva/elixir_sense) to extract and provide some of the language intelligence. + +As the Language Server Protocol adoption grew as a whole, so did the usage of Elixir's implementation, which eventually became the main mechanism Elixir users interacted with the language from their editors. + +Eventually, Elixir's language server implementation got its [own organization on GitHub](https://github.com/elixir-lsp/), and maintenance reins were given to Łukasz Samson and Jason Axelson. + +Over time, the Elixir Language Server has accrued technical debt. Some of it exists due to intrinsic complexities (for example, the Language Server Protocol uses UTF-16 for text encoding, instead of the more widely used UTF-8), while others are a consequence of working on codebase while both the programming language and the protocol themselves were still evolving. + +This led to Mitch Hanberg and Steve Cohen to create alternative language server implementations, exploring different trade-offs. + +For example, both [Next LS](https://github.com/elixir-tools/next-ls) and [Lexical](https://github.com/lexical-lsp/lexical) use Erlang Distribution to isolate the Language Server runtime from the user code. + +Next LS also focused on extracting the LSP protocol parts into [GenLSP](https://github.com/elixir-tools/gen_lsp) (which can be used by anyone to easily create a language server), single binary distribution with [Burrito](https://github.com/burrito-elixir/burrito), and experimenting with SQLite for the symbol index. + +[Lexical](https://github.com/lexical-lsp/lexical) concerned itself with speed and abstractions to deal with documents, ranges, and more. + +This means the Elixir community had, for some time, three distinct language server implementations, each with their own strengths. + +## Looking forward + +The current language server maintainers have agreed to move forward with a _single Language Server Protocol project_, relying on the strengths of each implementation: + +* Lexical provides a stable foundation +* ElixirLS, through ElixirSense, provides the most complete implementation and wider functionality +* Next LS, through GenLSP, provides a general interface for LSP implementations and straight-forward packaging via [Burrito](https://github.com/burrito-elixir/burrito) + +The above is a rough outline, as the specific details of how the projects will move forward are still being discussed. While some of the team members also maintain direct integration with some editors, we will continue relying on the community's help and efforts to get full coverage across all available editors. + +And there is still a lot more to do! + +Many underestimate the complexity behind implementing the Language Server Protocol. That's not surprising: we mostly interact with it from an editor, allowing us to freely ignore what makes it tick. + +In practice, the Language Server needs, in many ways, to reimplement several parts of the language and its compiler. + +If the Elixir compiler sees the code `some_value +`, it can immediately warn and say: "this expression is incomplete". However, the Language Server still needs to make sense of invalid code to provide features like completion. And that applies to everything: missing do-end blocks, invalid operators, invoking macros that do not exist, etc. Mitch has made [Spitfire](https://github.com/elixir-tools/spitfire), an error tolerant parser to tackle this particular problem. + +Some ecosystems have undertaken [multi-year efforts to redesign their compilers and toolchains](https://en.wikipedia.org/wiki/Roslyn_(compiler)) to provide better tools for lexical and semantic code analysis (which most likely took a significant investment of time and resources to conclude). That's to say some of the problems faced by Language Server implementations will be best tackled if they are also solved as part of Elixir itself. + +For example, every Language Server implementation compiles their own version of a project, making it so every application and its dependencies have to be compiled twice in development: once for Mix and once for the Language Server. Wouldn't it be nice if Elixir and the Language Servers could all rely on the same compilation artifacts? + +This is not news to the Elixir team either: almost every Elixir release within the last 3 years has shipped new code analysis APIs, such as [Code.Fragment](https://hexdocs.pm/elixir/Code.Fragment.html), with the goal of removing duplication across Language Servers, [IEx](https://hexdocs.pm/iex), and [Livebook](https://livebook.dev/), as well as reduce their reliance on internal Elixir modules. Most recently, Elixir v1.17 shipped with [new APIs to help developers emulate the compiler behaviour](https://hexdocs.pm/elixir/Macro.Env.html). Our goal is to make these building blocks available for all Elixir developers, so their benefits are reaped beyond the language server tooling. + +Furthermore, as [set-theoretic types make their way into Elixir](https://elixir-lang.org/blog/2024/06/12/elixir-v1-17-0-released/), we also want to provide official APIs to integrate them into our tools. + +## Sponsorships + +Currently, [Fly.io](https://fly.io/) is sponsoring Łukasz Samson to work part-time on the Language Server and editor integration. The [Livebook](https://livebook.dev/) project is donating development time from Jonatan Kłosko, creator of Livebook, to improve the Elixir compiler and its code intelligence APIs. + +We are grateful to both companies for investing into the community and you should check them out. + +As mentioned above, Language Server implementations are complex projects, and unifying efforts is an important step in the right direction. However, we also need community help, and one of the ways to do so is by sponsoring the developers making this possible: + +* [Łukasz Samson](https://github.com/sponsors/lukaszsamson) +* [Mitch Hanberg](https://github.com/sponsors/mhanberg) +* [Steve Cohen](https://github.com/sponsors/scohen) + +Companies who can afford to sponsor part-time development are welcome to reach out and help us achieve this important milestone. + +## Progress updates + +A new project website and social media accounts will be created soon, and you can follow them to stay up to date with our progress and any interesting developments. + +The name of the new project is still in the works as well as many of the decisions we'll need to make, so please have patience! + +In the meantime, you can continue to use the language server of your choice, and we’ll be sure to make the transition to the fourth and final project as smooth as possible. + +Thank you! diff --git a/_posts/2024-08-28-typing-lists-and-tuples.markdown b/_posts/2024-08-28-typing-lists-and-tuples.markdown new file mode 100644 index 000000000..545996655 --- /dev/null +++ b/_posts/2024-08-28-typing-lists-and-tuples.markdown @@ -0,0 +1,173 @@ +--- +layout: post +title: "Typing lists and tuples in Elixir" +authors: +- José Valim +category: Internals +excerpt: "This article explores the design decisions of typing lists and tuples in Elixir within a sound gradual type system" +--- + +We have been working on [a type system for the Elixir programming language](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/). The type system provides sound gradual typing: it can safely interface static and dynamic code, and if the program type checks, it will not produce type errors at runtime. + +It is important to emphasize **type errors**. The type systems used at scale today do not guarantee the absense of any runtime errors, but only typing ones. Many programming languages error when accessing the "head" of an empty list, most languages raise on division by zero or when computing the logarithm of negative numbers on a real domain, and others may fail to allocate memory or when a number overflows/underflows. + +Language designers and maintainers must outline the boundaries of what can be represented as typing errors and how that impacts the design of libraries. The goal of this article is to highlight some of these decisions in the context of lists and tuples in Elixir's on-going type system work. + +> In this article, the words "raise" and "exceptions" describe something unexpected happened, and not a mechanism for control-flow. Other programming languages may call them "panics" or "faults". + +## The `head` of a list + +Imagine you are designing a programming language and you want to provide a `head` function, which returns the head - the first element - of a list, you may consider three options. + +The first option, the one found in many programming languages, is to raise if an empty list is given. Its implementation in Elixir would be something akin to: + +```elixir +$ list(a) -> a +def head([head | _]), do: head +def head([]), do: raise "empty list" +``` + +Because the type system cannot differentiate between an empty list and a non-empty list, you won't find any typing violations at compile-time, but an error is raised at runtime for empty lists. + +An alternative would be to return an `option` type, properly encoding that the function may fail (or not): + +```elixir +$ list(a) -> option(a) +def head([head | _]), do: {:ok, head} +def head([]), do: :none +``` + +This approach may be a bit redundant. Returning an `option` type basically forces the caller to pattern match on the returned `option`. While many programming languages provide functions to compose `option` values, one may also get rid of the additional wrapping and directly pattern match on the list instead. So instead of: + +```elixir +case head(list) do + {:ok, head} -> # there is a head + :none -> # do what you need to do +end +``` + +You could just write: + +```elixir +case list do + [head | _] -> # there is a head + [] -> # do what you need to do +end +``` + +Both examples above are limited by the fact the type system cannot distinguish between empty and non-empty lists and therefore their handling must happen at runtime. If we get rid of this limitations, we could define `head` as follows: + +```elixir +$ non_empty_list(a) -> a +def head([head | _]), do: head +``` + +And now we get a typing violation at compile-time if an empty list is given as argument. There is no `option` tagging and no runtime exceptions. Win-win? + +The trouble with the above is that now it is responsibility of the language users to prove the list is not empty. For example, imagine this code: + +```elixir +list = convert_json_array_to_elixir_list(json_array_as_string) +head(list) +``` + +In the example above, since `convert_json_array_to_elixir_list` may return an empty list, there is a typing violation at compile-time. To resolve it, we need to prove the result of `convert_json_array_to_elixir_list` is not an empty list before calling `head`: + +```elixir +list = convert_json_array_to_elixir_list(json_array_as_string) + +if list == [] do + raise "empty list" +end + +head(list) +``` + +But, at this point, we might as well just use pattern matching and once again get rid of `head`: + +```elixir +case convert_json_array_to_elixir_list(json_array_as_string) do + [head | _] -> # there is a head + [] -> # do what you need to do +end +``` + +Most people would expect that encoding more information into the type system would bring only benefits but there is a tension here: the more you encode into types, the more you might have to prove in your programs. + +While different developers will prefer certain idioms over others, I am not convinced there is one clearly superior approach here. Having `head` raise a runtime error may be the most pragmatic approach _if_ the developer expects the list to be non-empty in the first place. Returning `option` gets rid of the exception by forcing users to explicitly handle the result, but leads to more boilerplate compared to pattern matching, especially if the user does not expect empty lists. And, finally, adding precise types means there could be more for developers to prove. + +### What about Elixir? + +Thanks to set-theoretic types, we will most likely distinguish between empty lists and non-empty lists in Elixir's type system, since pattern matching on them is a common language idiom. Furthermore, several functions in Elixir, such as `String.split/2` are guaranteed to return non-empty lists, which can then be nicely encoded into a function's return type. + +Elixir also has the functions `hd` (for head) and `tl` (for tail) inherited from Erlang, which are [valid guards](https://hexdocs.pm/elixir/patterns-and-guards.html). They only accept non-empty lists as arguments, which will now be enforced by the type system too. + +This covers almost all use cases but one: what happens if you want to access the first element of a list, which has not been proven to be empty? You could use pattern matching and conditionals for those cases, but as seen above, this can lead to common boilerplate such as: + +```elixir +if list == [] do + raise "unexpected empty list" +end +``` + +Luckily, it is common in Elixir to use the `!` suffix to encode the possibility of runtime errors for _valid_ inputs. For these circumstances, we may introduce `List.first!` (and potentially `List.drop_first!` for the tail variant). + +## Accessing tuples + +Now that we have discussed lists, we can talk about tuples. In a way, tuples are more challenging than lists for two reasons: + +1. A list is a collection where all elements have the same type (be it a `list(integer())` or `list(integer() or float())`), while tuples carry the types of each element + +2. We natively access tuples by index, instead of its head and tail, such `elem(tuple, 0)` + +In the upcoming v1.18 release, Elixir's new type system will support tuple types, and they are written between curly brackets. For example, the [`File.read/1` function](https://hexdocs.pm/elixir/File.html#read/1) would have the return type `{:ok, binary()} or {:error, posix()}`, quite similar to today's typespecs. + +The tuple type can also specify a minimum size, as you can also write: `{atom(), integer(), ...} `. This means the tuple has at least two elements, the first being an `atom()` and the second being an `integer()`. This definition is required for type inference in patterns and guards. After all, a guard `is_integer(elem(tuple, 1))` tells you the tuple has at least two elements, with the second one being an integer, but nothing about the other elements and the tuple overall size. + +With tuples support merged into main, we need to answer questions such as which kind of compile-time warnings and runtime exceptions tuple operations, such as `elem(tuple, index)` may emit. Today, we know that it raises if: + +1. the index is out of bounds, as in `elem({:ok, "hello"}, 3)` + +2. the index is negative, as in `elem({:ok, 123}, -1)` + +When typing `elem(tuple, index)`, one option is to use "avoid all runtime errors" as our guiding light and make `elem` return `option` types, such as: `{:ok, value}` or `:none`. This makes sense for an out of bounds error, but should it also return `:none` if the index is negative? One could argue that they are both out of bounds. On the other hand, a positive index may be correct depending on the tuple size but **a negative index is always invalid**. From this perspective, encoding an always invalid value as an `:none` can be detrimental to the developer experience, hiding logical bugs instead of (loudly) blowing up. + +Another option is to make these programs invalid. If we completely remove `elem/2` from the language and you can only access tuples via pattern matching (or by adding a literal notation such as `tuple.0`), then all possible bugs can be caught by the type checker. However, some data structures, such as [array in Erlang](https://www.erlang.org/doc/apps/stdlib/array.html) rely on dynamic tuple access, and implementing those would be no longer possible. + +Yet another option is to encode integers themselves as values in the type system. In the same way that Elixir's type system supports the values `:ok` and `:error` as types, we could support each integer, such as `13` and `-42` as types as well (or specific subsets, such as `neg_integer()`, `zero()` and `pos_integer()`). This way, the type system would know the possible values of `index` during type checking, allowing us to pass complex expressions to `elem(tuple, index)`, and emit typing errors if the indexes are invalid. However, remember that encoding more information into types may force developers to also prove that those indexes are within bounds in many other cases. + +Once again, there are different trade-offs, and we must select one that best fit into Elixir use and semantics today. + +### What about Elixir? + +The approach we are taking in Elixir is two-fold: + +* If the index is a literal integer, it will perform an exact access on the tuple element. This means `elem(tuple, 1)` will work if we can prove the tuple has at least size 2, otherwise you will have a type error + +* If the index is not a literal integer, the function will fallback to a dynamic type signature + +Let's expand on the second point. + +At a fundamental level, we could describe `elem` with the type signature of `tuple(a), integer() -> a`. However, the trouble with this signature is that it does not tell the type system (nor users) the possibility of a runtime error. Luckily, because Elixir will offer a gradual type system, we could encode the type signature as `dynamic({...a}), integer() -> dynamic(a)`. By encoding the argument and return type as dynamic, developers who want a fully static program will be notified of a typing error, while existing developers who rely on dynamic features of the language can continue to do so, and those choices are now encoded into the types. + +Overall, + +* For static programs (the ones that do not use the `dynamic()` type), `elem/2` will validate that the first argument is a tuple of known shape, and the second argument is a literal integer which is greater than or equal to zero and less than the tuple size. This guarantees no runtime exceptions. + +* Gradual programs will have the same semantics (and runtime exceptions) as today. + +## Summary + +I hope this article outlines some of the design decisions as we bring a gradual type system to Elixir. Although supporting tuples and lists is a "table stakes" feature in most type systems, bringing them to Elixir is an opportunity to understand how the type system will interact with several language idioms, as well as provide a foundation for future decisions. The most important take aways are: + +1. Type safety is a commitment from both sides. If you want your type system to find even more bugs through more precise types, you will need to prove more frequently that your programs are free of certain typing violations. + +2. It is not a goal of the type system to avoid all runtime errors. This would require either a type system that is too precise (and require more proofs) or it would require functions to mask hard errors (such as a negative index) as error values. Exceptions still play an important role in typed Elixir and, given they are modelled as structs, they will also be typed in the future. + +3. Elixir's convention of using the suffix `!` to provide variants that encode the possibility of runtime exceptions for a valid domain (the input types) nicely complements the type system, as it can help static programs avoid the boilerplate of converting `:none`/`:error` into exceptions for unexpected scenarios. + +4. Using `dynamic()` in function signatures is a mechanism available in Elixir's type system to signal that a function has dynamic behaviour and may raise runtime errors, allowing violations to be reported on programs that wish to remain fully static. Similar to how other static languages provide dynamic behaviour via `Any` or `Dynamic` types. + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), [Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +Happy typing! diff --git a/_posts/2024-12-19-elixir-v1-18-0-released.markdown b/_posts/2024-12-19-elixir-v1-18-0-released.markdown new file mode 100644 index 000000000..22edc0364 --- /dev/null +++ b/_posts/2024-12-19-elixir-v1-18-0-released.markdown @@ -0,0 +1,218 @@ +--- +layout: post +title: "Elixir v1.18 released: type checking of calls, LSP listeners, built-in JSON, and more" +authors: +- José Valim +category: Releases +excerpt: "Elixir v1.18 released: type checking of function calls, Language Server listeners, built-in JSON, ExUnit improvements, mix format --migrate, and more" +--- + +Elixir v1.18 is an impressive release with improvements across the two main efforts happening within the Elixir ecosystem right now: set-theoretic types and language servers. It also comes with built-in JSON support and adds new capabilities to its unit testing library. Let's go over each of those in detail. + +## Type inference of patterns and return types + +There are several updates in the typing department, so let's break them down. + +#### A type system? In my Elixir? + +There is an on-going [research and development](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/) effort to bring static types to Elixir. Elixir's type system is: + + * **sound** - the types inferred and assigned by the type system align with the behaviour of the program + + * **gradual** - Elixir's type system includes the `dynamic()` type, which can be used when the type of a variable or expression is checked at runtime. In the absence of `dynamic()`, Elixir's type system behaves as a static one + + * **developer friendly** - the types are described, implemented, and composed using basic set operations: unions, intersections, and negation (hence it is a set-theoretic type system) + +More interestingly, you can compose `dynamic()` with any type. For example, `dynamic(integer() or float())` means the type is either `integer()` or `float()` at runtime. This allows the type system to emit warnings if none of the types are satisfied, even in the presence of dynamism. + +#### What has already been done? + +[Elixir v1.17 was the first release to incorporate the type system in the compiler](https://elixir-lang.org/blog/2024/06/12/elixir-v1-17-0-released/). In particular, we have added support for primitive types (integer, float, binary, pids, references, ports), atoms, and maps. We also added type checking to a handful of operations related to those types, such as accessing fields in maps, as in `user.adress` (mind the typo), performing structural comparisons between structs, as in `my_date < ~D[2010-04-17]`, etc. + +#### What is new in v1.18? + +The most exciting change in Elixir v1.18 is type checking of function calls, alongside gradual inference of patterns and return types. To understand how this will impact your programs, consider the following code defined in `lib/user.ex`: + +```elixir +defmodule User do + defstruct [:age, :car_choice] + + def drive(%User{age: age, car_choice: car}, car_choices) when age >= 18 do + if car in car_choices do + {:ok, car} + else + {:error, :no_choice} + end + end + + def drive(%User{}, _car_choices) do + {:error, :not_allowed} + end +end +``` + +Elixir's type system will infer the `drive` function expects a `User` struct as input and returns either `{:ok, dynamic()}` or `{:error, :no_choice}` or `{:error, :not_allowed}`. Therefore, the following code + +```elixir +User.drive({:ok, %User{}}, car_choices) +``` + +will emit a warning stating that we are passing an invalid argument: + +![Example of a warning when passing wrong argument to a function](/images/contents/type-warning-function-clause.png) + +Now consider the expression below. We are expecting the `User.drive/2` call to return `:error`, which cannot possibly be true: + +```elixir +case User.drive(user, car_choices) do + {:ok, car} -> car + :error -> Logger.error("User cannot drive") +end +``` + +Therefore the code above would emit the following warning: + +![Example of a warning when a case clause won't ever match](/images/contents/type-warning-case.png) + +Our goal is for the warnings to provide enough contextual information that lead to clear reports and that's an area we are actively looking for feedback. If you receive a warning that is unclear, please open up a bug report. + +Elixir v1.18 also augments the type system with support for tuples and lists, plus type checking of almost all Elixir language constructs, except `for`-comprehensions, `with`, and closures. Here is a non-exaustive list of the new violations that can be detected by the type system: + + * if you define a pattern that will never match any argument, such as `def function(x = y, x = :foo, y = :bar)` + + * matching or accessing tuples at an invalid index, such as `elem(two_element_tuple, 2)` + + * if you have a branch in a `try` that will never match the given expression + + * if you have a branch in a `cond` that always passes (except the last one) or always fails + + * if you attempt to use the return value of a call to `raise/2` (which by definition returns no value) + +In summary, this release takes us further in our journey of providing type checking and type inference of existing Elixir programs, without requiring Elixir developers to explicitly add type annotations. + +For existing codebases with reasonable code coverage, most type system reports will come from uncovering dead code - code which won't ever be executed - as seen in a [few](https://github.com/phoenixframework/phoenix_live_view/commit/6c6e2aaf6a01957cc6bb8a27d2513bff273e8ca2) [distinct](https://github.com/elixir-ecto/postgrex/commit/3308f277f455ec64f2d0d7be6263f77f295b1325) [projects](https://github.com/phoenixframework/flame/commit/0c0c2875e42952d2691cbdb7928fc32f4715e746). A notable example is the type system ability to track how private functions are used throughout a module and then point out which clauses are unused: + +```elixir +defmodule Example do + def public(x) do + private(Integer.parse(x)) + end + + defp private(nil), do: nil + defp private("foo"), do: "foo" + defp private({int, _rest}), do: int + defp private(:error), do: 0 + defp private("bar"), do: "bar" +end +``` + +![Example of a warning for unused private clauses](/images/contents/type-warning-private.png) + +Keep in mind the current implementation does not perform type inference of guards yet, which is an important source of typing information in programs. There is a lot the type system can learn about our codebases, that it does not yet. This brings us to the next topic. + +#### Future work + +The next Elixir release should improve the typing of maps, tuples, and closures, allowing us to type even more constructs. We also plan to fully type the `with` construct, `for`-comprehensions, as well as protocols. + +But more importantly, we want to focus on complete type inference of guards, which in turn will allow us to explore ideas such as redundant pattern matching clauses and exhaustiveness checks. Our goal with inference is to strike the right balance between developer experience, compilation times, and the ability of finding provable errors in existing codebases. You can learn more [about the trade-offs we made for inference in our documentation](https://hexdocs.pm/elixir/1.18/gradual-set-theoretic-types.html#type-inference). + +Future Elixir versions will introduce user-supplied type signatures, which should bring the benefits of a static type system without relying on inference. [Check our previous article on the overall milestones for more information](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/). + +#### Sponsors + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/) ([they are hiring!](https://www.fresha.com/careers/openings?department=engineering)), [Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +## Language server listeners + +Three months ago, we welcomed [the Official Language Server team](https://elixir-lang.org/blog/2024/08/15/welcome-elixir-language-server-team/), with the goal of unifying the efforts behind code intelligence, tools, and editors in Elixir. Elixir v1.18 brings new features on this front by introducing locks and listeners to its compilation. Let's understand what it means. + +At the moment, all language server implementations have their own compilation environment. This means that your project and dependencies during development are compiled once, for your own use, and then again for the language server. This duplicate effort could cause the language server experience to lag, when it could be relying on the already compiled artifacts of your project. + +This release addresses the issue by introducing a compiler lock, ensuring that only a single operating system running Elixir compiles your project at a given moment, and by providing the ability for one operating system process to listen to the compilation results of others. In other words, different Elixir instances can now communicate over the same compilation build, instead of racing each other. + +These enhancements do not only improve editor tooling, but they also directly benefit projects like IEx and Phoenix. Here is a quick snippet showing how to enable auto-reloading inside IEx, then running `mix compile` in one shell automatically reloads the module inside the IEx session: + + + +## Built-in JSON + +[Erlang/OTP 27 added built-in support for JSON](https://www.erlang.org/doc/apps/stdlib/json.html) and we are now bringing it to Elixir. A new module, called [`JSON`](https://hexdocs.pm/elixir/1.18/JSON.html), has been added with functions to encode and decode JSON. Its most basic APIs reflect the ones [from the Jason project](https://hexdocs.pm/jason/Jason.html) (the de-facto JSON library in the Elixir community up to this point). + +A new protocol, called [`JSON.Encoder`](https://hexdocs.pm/elixir/1.18/JSON.Encoder.html), is also provided for those who want to customize how their own data types are encoded to JSON. You can also derive protocols for structs, with a single-line of code: + +```elixir +@derive {JSON.Encoder, only: [:id, :name]} +defstruct [:id, :name, :email] +``` + +The deriving API mirrors the one from `Jason`, helping those who want to migrate to the new `JSON` module. + +## Parameterized tests and ExUnit groups + +[ExUnit now supports parameterized tests](https://hexdocs.pm/ex_unit/1.18/ExUnit.Case.html#module-parameterized-tests). This allows your test modules to run multiple times under different parameters. + +For example, Elixir ships [a local, decentralized and scalable key-value process storage called `Registry`](https://hexdocs.pm/elixir/Registry.html). The registry can be partitioned and its implementation differs depending if partitioning is enabled or not. Therefore, during tests, we want to ensure both modes are exercised. With Elixir v1.18, we can achieve this by writing: + +```elixir +defmodule Registry.Test do + use ExUnit.Case, + async: true, + parameterize: [ + %{partitions: 1}, + %{partitions: 8} + ] + + # ... the actual tests ... +end +``` + +Once specified, the number of partitions is available as part of the test configuration. For example, to start one registry per test with the correct number of partitions, you can write: + +```elixir + setup config do + partitions = config.partitions + name = :"#{config.test}_#{partitions}" + opts = [keys: :unique, name: name, partitions: partitions] + start_supervised!({Registry, opts}) + opts + end +``` + +Prior to parameterized tests, Elixir resorted on code generation, which increased compilation times. Furthermore, ExUnit parameterizes the whole test modules, which also allows the different parameters to run concurrently if the `async: true` option is given. Overall, this features allows you to compile and run multiple scenarios more efficiently. + +Finally, ExUnit also comes with the ability of specifying test groups. While ExUnit supports running tests concurrently, those tests must not have shared state between them. However, in large applications, it may be common for some tests to depend on some shared state, and other tests to depend on a completely separate state. For example, part of your tests may depend on Cassandra, while others depend on Redis. Prior to Elixir v1.18, these tests could not run concurrently, but in v1.18 they might as long as they are assigned to different groups: + +```elixir +defmodule MyApp.PGTest do + use ExUnit.Case, async: true, group: :pg + + # ... +end +``` + +Tests modules within the same group do not run concurrently, but across groups, they might. + +With features like async tests, suite partitioning, and now grouping, Elixir developers have plenty of flexibility to make the most use of their machine resources, both in development and in CI. + +## `mix format --migrate` + +The `mix format` command now supports an explicit `--migrate` flag, which will convert constructs that have been deprecated in Elixir to their latest version. Because this flag rewrites the AST, it is not guaranteed the migrated format will always be valid when used in combination with macros that also perform AST rewriting. + +As of this release, the following migrations are executed: + + * Normalize parens in bitstring modifiers - it removes unnecessary parentheses in known bitstring modifiers, for example `<>` becomes `<>`, or adds parentheses for custom modifiers, where `<>` becomes `<>`. + + * Charlists as sigils - formats charlists as `~c` sigils, for example `'foo'` becomes `~c"foo"`. + + * `unless` as negated `if`s - rewrites `unless` expressions using `if` with a negated condition, for example `unless foo do` becomes `if !foo do`. We plan to deprecate `unless` in future releases. + +More migrations will be added in future releases to help us push towards more consistent codebases. + +## Summary + +Other notable changes include [`PartitionSupervisor.resize!/2`](https://hexdocs.pm/elixir/1.18/PartitionSupervisor.html#resize!/2), for resizing the number of partitions (aka processes) of a supervisor at runtime, [Registry.lock/3](https://hexdocs.pm/elixir/1.18/Registry.html#lock/3) for simple in-process key locks, PowerShell versions of `elixir` and `elixirc` scripts for better DX on Windows, and more. [See the CHANGELOG](https://hexdocs.pm/elixir/1.18/changelog.html) for the complete release notes. + +Happy coding! diff --git a/_posts/2025-01-21-remote-elixir-case.markdown b/_posts/2025-01-21-remote-elixir-case.markdown new file mode 100644 index 000000000..10d47c32d --- /dev/null +++ b/_posts/2025-01-21-remote-elixir-case.markdown @@ -0,0 +1,111 @@ +--- +layout: post +title: "Remote: growing from zero to unicorn with Elixir" +authors: +- Hugo Baraúna +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Remote. +flagship: true +logo: /images/cases/logos/remote.png +tags: growth team web +--- + +*Welcome to our series of [case studies about companies using Elixir in production](/cases.html).* + +Remote is the everywhere employment platform enabling companies to find, hire, manage, and pay people anywhere across the world. + +Founded in 2019, they reached unicorn status in just over two years and have continued their rapid growth trajectory since. + +Since day zero, Elixir has been their primary technology. Currently, their engineering organization as a whole consists of nearly 300 individuals. + +This case study focuses on their experience using Elixir in a high-growth environment. + +![Remote website screenshot](/images/cases/bg/remote.png) + +## Why Elixir? + +Marcelo Lebre, co-founder and president of Remote, had worked with many languages and frameworks throughout his career, often encountering the same trade-off: easy-to-code versus easy-to-scale. + +In 2015, while searching for alternatives, he discovered Elixir. Intrigued, Marcelo decided to give it a try and immediately saw its potential. At the time, Elixir was still in its early days, but he noticed how fast the community was growing, with support for packages and frameworks starting to show up aggressively. + +In December 2018, when Marcelo and his co-founder decided to start the company, they had to make a decision about the technology that would support their vision. Marcelo wanted to prioritize building a great product quickly without worrying about scalability issues from the start. He found Elixir to be the perfect match: + +> I wanted to focus on building a great product fast and not really worry about its scalability. Elixir was the perfect match—reliable performance, easy-to-read syntax, strong community, and a learning curve that made it accessible to new hires. +> +> \- Marcelo Lebre, Co-founder and President + +The biggest trade-off Marcelo identified was the smaller pool of Elixir developers compared to languages like Ruby or Python. However, he quickly realized that the quality of candidates more than made up for it: + +> The signal-to-noise ratio in the quality of Elixir candidates was much higher, which made the trade-off worthwhile. +> +> \- Marcelo Lebre, Co-founder and President + +## Growing with a monolithic architecture + +Remote operates primarily with a monolith, with Elixir in the backend and React in the front-end. + +The monolith enabled speed and simplicity, allowing the team to iterate quickly and focus on building features. However, as the company grew, they needed to invest in tools and practices to manage almost 180 engineers working in the same codebase. + +One practice was boundary enforcement. They used the [Boundary library](https://github.com/sasa1977/boundary) to maintain strict boundaries between modules and domains inside the codebase. + +Another key investment was optimizing their compilation time in the CI pipeline. Since their project has around 15,000 files, compiling it in every build would take too long. So, they implemented incremental builds in their CI pipeline, recompiling only the files affected by changes instead of the entire codebase. + +> I feel confident making significant changes in the codebase. The combination of using a functional language and our robust test suite allows us to keep moving forward without too much worry. +> +> \- André Albuquerque, Staff Engineer + +Additionally, as their codebase grew, the Elixir language continued to evolve, introducing better abstractions for developers working with large codebases. For example, with the release of Elixir v1.11, the [introduction of config/runtime.exs](/blog/2020/10/06/elixir-v1-11-0-released/) provided the Remote team with a better foundation for managing configuration. This enabled them to move many configurations from compile-time to runtime, significantly reducing unnecessary recompilations caused by configuration updates. + +## Infra-structure and operations + +One might expect Remote’s infrastructure to be highly complex, given their global scale and the size of their engineering team. Surprisingly, their setup remains relatively simple, reflecting a thoughtful balance between scalability and operational efficiency. + +Remote runs on AWS, using EKS (Elastic Kubernetes Service). The main backend (the monolith) operates in only five pods, each with 10 GB of memory. They use [Distributed Erlang](https://www.erlang.org/doc/system/distributed.html) to connect the nodes in their cluster, enabling seamless communication between processes running on different pods. + +For job processing, they rely on [Oban](https://github.com/oban-bg/oban), which runs alongside the monolith in the same pods. + +Remote also offers a public API for partners. While this API server runs separately from the monolith, it is the same application, configured to start a different part of its supervision tree. The separation was deliberate, as the team anticipated different load patterns for the API and wanted the flexibility to scale it independently. + +The database setup includes a primary PostgreSQL instance on AWS RDS, complemented by a read-replica for enhanced performance and scalability. Additionally, a separate Aurora PostgreSQL instance is dedicated to storing Oban jobs. Over time, the team has leveraged tools like PG Analyze to optimize performance, addressing bottlenecks such as long queries and missing indexes. + +This streamlined setup has proven resilient, even during unexpected spikes in workload. The team shared an episode where a worker’s job count unexpectedly grew by two orders of magnitude. Remarkably, the system handled the increase seamlessly, continuing to run as usual without requiring any design changes or manual intervention. + +> We once noticed two weeks later that a worker’s load had skyrocketed. But the scheduler worked fine, and everything kept running smoothly. That was fun. +> +> \- Alex Naser, Staff Engineer + +## Team organization and responsibilities + +Around 90% of their backend team works in the monolith, while the rest work in a few satellite services, also written in Elixir. + +Within the monolith, teams are organized around domains such as onboarding, payroll, and billing. Each team owns one or multiple domains. + +To streamline accountability in a huge monolith architecture, Remote invested heavily in team assignment mechanisms. + +They implemented a tagging system that assigns ownership down to the function level. This means any trace—whether sent to tools like Sentry or Datadog—carries a tag identifying the responsible team. This tagging also extends to endpoints, allowing teams to monitor their areas effectively and even set up dashboards for alerts, such as query times specific to their domain. + +The tagging system also simplifies CI workflows. When a test breaks, it’s automatically linked to the responsible team based on the Git commit. This ensures fast issue identification and resolution, removing the need for manual triaging. + +## Hiring and training + +Remote’s hiring approach prioritizes senior engineers, regardless of their experience with Elixir. + +During the hiring process, all candidates are required to complete a coding exercise in Elixir. For those unfamiliar with the language, a tailored version of the exercise is provided, designed to introduce them to Elixir while reflecting the challenges they would face if hired. + +Once hired, new engineers are assigned an engineering buddy to guide them through the onboarding process. + +For hires without prior Elixir experience, Remote developed an internal Elixir training camp, a curated collection of best practices, tutorials, and other resources to introduce new hires to the language and ecosystem. This training typically spans two to four weeks. + +After completing the training, engineers are assigned their first tasks—carefully selected tickets designed to build confidence and familiarity with the codebase. + +## Summing up + +Remote’s journey highlights how thoughtful technology, infrastructure, and team organization decisions can support rapid growth. + +By leveraging Elixir’s strengths, they built a monolithic architecture that balanced simplicity with scalability. This approach allowed their engineers to iterate quickly in the early stages while effectively managing the complexities of a growing codebase. + +Investments in tools like the Boundary library and incremental builds ensured their monolith remained efficient and maintainable even as the team and codebase scaled dramatically. + +Remote's relatively simple infrastructure demonstrates that scaling doesn't always require complexity. Their ability to easily handle unexpected workload spikes reflects the robustness of their architecture and operational practices. + +Finally, their focus on team accountability and streamlined onboarding allowed them to maintain high productivity while integrating engineers from diverse technical backgrounds, regardless of their prior experience with Elixir. diff --git a/_posts/2025-02-26-elixir-openchain-certification.markdown b/_posts/2025-02-26-elixir-openchain-certification.markdown new file mode 100644 index 000000000..76285ea70 --- /dev/null +++ b/_posts/2025-02-26-elixir-openchain-certification.markdown @@ -0,0 +1,72 @@ +--- +layout: post +title: "Announcing Elixir OpenChain Certification" +authors: + - Jonatan Männchen + - José Valim +category: Announcements +excerpt: "The Elixir project now meets OpenChain (ISO/IEC 5230). Each release ships with Source SBoMs in CycloneDX 1.6 and SPDX 2.3, plus attestation." +tags: openchain compliance +--- + +We are pleased to share that the Elixir project now complies with +[OpenChain][openchain] ([ISO/IEC 5230][iso_5230]), an international +standard for open source license compliance. This step aligns with broader +efforts to meet industry standards for supply chain and cybersecurity best +practices. + +“Today’s announcement around Elixir’s conformance represents another significant +example of community maturity,” says Shane Coughlan, OpenChain General Manager. +“With projects - the final upstream - using ISO standards for compliance and +security with increasing frequency, we are seeing a shift to longer-term +improvements to trust in the supply chain.” + +## Why OpenChain Compliance Helps + +By following OpenChain (ISO/IEC 5230), we demonstrate clear processes around +license compliance. This benefits commercial and community users alike, making +Elixir easier to adopt and integrate with confidence. + +## Changes for Elixir Users + +Elixir has an automated release process where its artifacts are signed. This +change strengthens this process by: + +- All future Elixir releases will include a Source SBoM in + [CycloneDX 1.6 or later][cyclonedx] and [SPDX 2.3 or later][spdx] formats. +- Each release will be attested along with the Source SBoM. + +These additions offer greater transparency into the components and licenses of +each release, supporting more rigorous supply chain requirements. + +## Changes for Contributors + +Contributing to Elixir remains largely the same, we have added more clarity and +guidelines around it: + +- Contributions remain under the Apache-2.0 License. Other licenses cannot be + accepted. +- The project now enforces the [Developer Certificate of Origin (DCO)][dco], + ensuring clarity around contribution ownership. + +Contributors will notice minimal procedural changes, as standard practices +around licensing remain in place. + +For more details, see the [CONTRIBUTING guidelines][contributing]. + +## Commitment + +These updates were made in collaboration with the +[Erlang Ecosystem Foundation][erlef], reflecting a shared +commitment to robust compliance and secure development practices. Thank you to +everyone who supported this milestone. We appreciate the community’s ongoing +contributions and look forward to continuing the growth of Elixir under these +established guidelines. + +[openchain]: https://openchainproject.org/ +[erlef]: https://erlef.org/ +[spdx]: https://spdx.org/rdf/terms/ +[cyclonedx]: https://cyclonedx.org/specification/overview/ +[iso_5230]: https://www.iso.org/standard/81039.html +[dco]: https://developercertificate.org/ +[contributing]: https://github.com/elixir-lang/elixir/blob/main/CONTRIBUTING.md diff --git a/_posts/2025-03-25-cyanview-elixir-case.markdown b/_posts/2025-03-25-cyanview-elixir-case.markdown new file mode 100644 index 000000000..48c021eae --- /dev/null +++ b/_posts/2025-03-25-cyanview-elixir-case.markdown @@ -0,0 +1,92 @@ +--- +layout: post +title: "Cyanview: Coordinating Super Bowl's visual fidelity with Elixir" +authors: +- Lars Wikman +- José Valim +category: Elixir in Production +excerpt: A case study of how Elixir is being used at Cyanview. +logo: /images/cases/logos/cyanview.png +tags: superbowl mqtt +--- + +How do you coordinate visual fidelity across two hundred cameras for a live event like the Super Bowl? + +The answer is: by using the craft of camera shading, which involves adjusting each camera to ensure they match up in color, exposure and various other visual aspects. The goal is to turn the live event broadcast into a cohesive and consistent experience. For every angle used, you want the same green grass and the same skin tones. Everything needs to be very closely tuned across a diverse set of products and brands. From large broadcast cameras, drone cameras, and PTZ cameras to gimbal-held mirrorless cameras and more. This is what Cyanview does. Cyanview is a small Belgian company that sells products for the live video broadcast industry, and its primary focus is shading. + +Broadcast is a business where you only get one chance to prove that your tool is up to the task. Reliability is king. There can be no hard failures. + +A small team of three built a product so powerful and effective that it spread across the industry purely on the strength of its functionality. Without any marketing, it earned a reputation among seasoned professionals and became a staple at the world’s top live events. Cyanview's Remote Control Panel (RCP) is now used by specialist video operators on the Olympics, Super Bowl, NFL, NBA, ESPN, Amazon and many more. Even most fashion shows in Paris use Cyanview’s devices. + +These devices put Elixir right in the critical path for serious broadcast operations. By choosing Elixir, Cyanview gained best-in-class networking features, state-of-the-art resilience and an ecosystem that allowed fast iteration on product features. + +![Operating many displays with Cyanview products](/images/cases/bg/cyanview-4.jpg "Operating many displays with Cyanview products") + +## Why Elixir? + +The founding team of Cyanview primarily had experience with embedded development, and the devices they produce involve a lot of low-level C code and plenty of FPGA. This is due to the low-level details of color science and the really tight timing requirements. + +If you’ve ever worked with camera software, you know it can be a mixed bag. Even after going fully digital, much of it remained tied to analog systems or relied on proprietary connectivity solutions. Cyanview has been targeting IP (as in Internet Protocol) from early on. This means Cyanview's software can operate on commodity networks that work in well-known and well-understood ways. This has aligned well with an increase in remote production, partially due to the pandemic, where production crews operate from a central location with minimal crew on location. Custom radio frequency or serial wire protocols have a hard time scaling to cross-continent distances. + +This also paved the way for Elixir, as the Erlang VM was designed to communicate and coordinate millions of devices, reliably, over the network. + +Elixir was brought in by the developer Ghislain, who needed to build integrations with cameras and interact with the other bits of required video gear, with many different protocols over the network. The language comes with a lot of practical features for encoding and decoding binary data down to the individual bits. Elixir gave them a strong foundation and the tools to iterate fast. + +Ghislain has been building the core intellectual property of Cyanview ever since. While the physical device naturally has to be solid, reliable, and of high quality, a lot of the secret sauce ultimately lies in the massive number of integrations and huge amounts of reverse engineering. Thus, the product is able to work with as many professional camera systems and related equipment as possible. It is designed to be compatible with everything and anything a customer is using. Plus, it offers an API to ensure smooth integration with other devices. + +David Bourgeois, the founder of Cyanview, told us a story how these technical decisions alongside Elixir helped them tackle real-world challenges: + +"During the Olympics in China, a studio in Beijing relied on a large number of Panasonic PTZ cameras. Most of their team, however, was based in Paris and needed to control the cameras remotely to run various shows throughout the day. The problem? Panasonic’s camera protocols were never designed for internet use — they require precise timing and multiple messages for every adjustment. With network latency, that leads to timeouts, disconnects, and system failures... So they ended up placing our devices next to the cameras in Beijing and controlled them over IP from Paris — just as designed." + +![Cyanview RIO device mounted on a camera at a sports field](/images/cases/bg/cyanview-2.jpg "Cyanview RIO device mounted on a camera at a sports field") + +The devices in a given location communicate and coordinate on the network over a custom MQTT protocol. Over a hundred cameras without issue on a single Remote Control Panel (RCP), implemented on top of Elixir's network stack. + +## Technical composition + +The system as a whole consists of RCP devices running a Yocto Linux system, with most of the logic built in Elixir and C. While Python is still used for scripting and tooling, its role has gradually diminished. The setup also includes multiple microcontrollers and the on-camera device, all communicating over MQTT. Additionally, cloud relays facilitate connectivity, while dashboards and controller UIs provide oversight and control. The two critical devices are the RCP offering control on the production end and the RIO handling low-latency manipulation of the camera. Both run Elixir. + +The configuration UI is currently built in Elm, but - depending on priorities - it might be converted to [Phoenix LiveView](https://phoenixframework.org/) over time to reduce the number of languages in use. The controller web UI is already in LiveView, and it is performing quite well on a very low-spec embedded Linux machine. + +The cloud part of the system is very limited today, which is unusual in a world of SaaS. There are cloud relays for distributing and sharing camera control as well as forwarding network ports between locations and some related features, also built in Elixir, but cloud is not at the core of the business. The devices running Elixir on location form a cluster over IP using a custom MQTT-based protocol suited to the task and are talking to hundreds of cameras and other video devices. + +It goes without saying that integration with so much proprietary equipment comes with challenges. Some integrations are more reliable than others. Some devices are more common, and their quirks are well-known through hard-won experience. A few even have good documentation that you can reference while others offer mystery and constant surprises. In this context, David emphasizes the importance of Elixir's mechanisms for recovering from failures: + +"If one camera connection has a blip, a buggy protocol or the physical connection to a device breaks it is incredibly important that everything else keeps working. And this is where Elixir’s supervision trees provide a critical advantage." + +## Growth & team composition + +The team has grown over the 9 years that the company has been operating, but it did so at a slow and steady pace. On average, the company has added just one person per year. With nine employees at the time of writing, Cyanview supports some of the biggest broadcast events in the world. + +There are two Elixir developers on board: Daniil who is focusing on revising some of the UI as well as charting a course into more cloud functionality, and Ghislain, who works on cameras and integration. Both LiveView and Elm are used to power device UIs and dashboards. + +What’s interesting is that, overall, the other embedded developers say that they don't know much about Elixir and they don't use it in their day-to-day work. Nonetheless, they are very comfortable implementing protocols and encodings in Elixir. The main reason they haven’t fully learned the language is simply time — they have plenty of other work to focus on, and deep Elixir expertise hasn’t been necessary. After all, there’s much more to their work beyond Elixir: designing PCBs, selecting electronic components, reverse engineering protocols, interfacing with displays, implementing FPGAs, managing production tests, real productions and releasing firmware updates. + +## Innovation and customer focus + +![Operator using Cyanview RCP for a massive crowd in an arena](/images/cases/bg/cyanview-3.jpg "Operator using Cyanview RCP for a massive crowd in an arena") + +Whether it’s providing onboard cameras in 40+ cars during the 24 hours of Le Mans, covering Ninja Warrior, the Australian Open, and the US Open, operating a studio in the Louvre, being installed in NFL pylons, or connecting over 200 cameras simultaneously – the product speaks for itself. Cyanview built a device for a world that runs on top of IP, using Elixir, a language with networking and protocols deep in its bones. This choice enabled them to do both: implement support for all the equipment and provide features no one else had. + +By shifting from conventional local-area radio frequency, serial connections, and inflexible proprietary protocols to IP networking, Cyanview’s devices redefined how camera systems operate. Their feature set is unheard of in the industry: Unlimited multicam. Tally lights. Pan & Tilt control. Integration with color correctors. World-spanning remote production. + +The ease and safety of shipping new functionality have allowed the company to support new features very quickly. One example is the increasing use of mirrorless cameras on gimbals to capture crowd shots. Cyanview were able to prototype gimbal control, test it with a customer and validate that it worked in a very short amount of time. This quick prototyping and validation of features is made possible by a flexible architecture that ensures that critical fundamentals don't break. + +Camera companies that don't produce broadcast shading remotes, such as Canon or RED, recommend Cyanview to their customers. Rather than competing with most broadcast hardware companies, Cyanview considers itself a partner. The power of a small team, a quality product and powerful tools can be surprising. Rather than focusing on marketing, Cyanview works very closely with its customers by supporting the success of their events and providing in-depth customer service. + +## Looking back and forward + +When asked if he would choose Elixir again, David responded: + +"Yes. We've seen what the Erlang VM can do, and it has been very well-suited to our needs. You don't appreciate all the things Elixir offers out of the box until you have to try to implement them yourself. It was not pure luck that we picked it, but we were still lucky. Elixir turned out to bring a lot that we did not know would be valuable to us. And we see those parts clearly now." + +Cyanview hopes to grow the team more, but plans to do so responsibly over time. Currently there is a lot more to do than the small team can manage. + +Development is highly active, with complementary products already in place alongside the main RCP device, and the future holds even more in that regard. Cloud offerings are on the horizon, along with exciting hardware projects that build on the lessons learned so far. As these developments unfold, we’ll see Elixir play an increasingly critical role in some of the world’s largest live broadcasts. + +![Cyanview Remote Control Panels in a control room](/images/cases/bg/cyanview-1.jpg "Cyanview Remote Control Panels in a control room") + +## In summary + +A high-quality product delivering the right innovation at the right time in an industry that's been underserved in terms of good integration. Elixir provided serious leverage for developing a lot of integrations with high confidence and consistent reliability. In an era where productivity and lean, efficient teams are everything, Cyanview is a prime example of how Elixir empowers small teams to achieve an outsized impact. + diff --git a/_posts/2025-06-02-elixir-outreach-stipend-for-speakers.markdown b/_posts/2025-06-02-elixir-outreach-stipend-for-speakers.markdown new file mode 100644 index 000000000..1bca122fc --- /dev/null +++ b/_posts/2025-06-02-elixir-outreach-stipend-for-speakers.markdown @@ -0,0 +1,42 @@ +--- +layout: post +title: "Elixir Outreach stipend for speakers and trainers" +authors: +- José Valim +category: Announcements +excerpt: "We announce the Elixir Outreach stipend in partnership with Dashbit, Oban, and the Erlang Ecosystem Foundation" +--- + +[Dashbit](https://dashbit.co), [Oban](https://oban.pro), and the [Erlang Ecosystem Foundation (EEF)](https://erlef.org) are glad to announce a new program, which we will trial over the next 12 months, called "Elixir Outreach". Our goal is to provide funds to community members who want to present Elixir and Erlang to other ecosystems and communities, while respecting our joint values. + +In a nutshell: + +* We will provide funds to community members to speak **in-person** about anything related to Elixir and the Erlang VM. + +* We will cover hotel and transportation costs for up to $700 USD. Please reach out, even if you expect to exceed that limit. This is our first time running the program and we're refining the budget. + +* The event must expect at least 150 attendees and happen outside of the Elixir, overall BEAM, and functional programming communities. In other words, we won't cover costs for attending Erlang, Elixir, or other BEAM/FP conferences nor meetups. Consider it as an opportunity to learn and bring external knowledge and experiences to the BEAM community. + +* You will be expected to send a report about your experience. The format and duration is up to you. We'd prefer that you write a blog post or an article sharing your overall experience with the involved communities. However, if you would prefer to only send it privately to us, that's fine too! + +The event should take place within your area. Our overall goal is to support multiple individuals, rather than drain our budget on a few long-distance flights (such as across coasts or continents). We are flexible on event location, distance, or type. If in doubt, reach out to [elixir_outreach at erlef dot org](mailto:elixir_outreach@erlef.org) + +Our initial budget of $7000 was donated by Dashbit ($5000) and Oban ($2000) to the Erlang Ecosystem Foundation (EEF), specifically for this program. The EEF will oversee the distribution of the funds. + +## Requesting a stipend + +To request a stipend, visit the [Erlang Ecosystem Foundation website and choose "Elixir Outreach" as the stipend type](https://erlef.org/stipends/form?type=elixir-outreach). + +Given we have limited funds, we cannot guarantee they will be available when you request them. We recommend reaching out to us before submitting or acceptance your talk. Therefore, by contacting us early, we can validate if the event matches the criteria above, ask questions, and earmark the funds. Once your talk is accepted, send us any itemized travel and accommodation costs so we can transfer the stipend to you, (not in excess of $700 USD). + +You can also request a stipend after your talk has already been accepted, but then there are no guarantees a stipend will be available. + +Our goal is to make this process simple and as straight-forward as possible. Although, we reserve the right to refuse a request for any reason. If in doubt, reach out to [elixir_outreach at erlef dot org](mailto:elixir_outreach@erlef.org). + +## Acknowledgements + +This is a new effort for all involved! Please be patient while we figure out the details. + +If you are looking for conferences to speak at, [Dave Aronson keeps a list of CFPs closing soon](https://www.codosaur.us/speaking/cfps-ending-soon) and there are likely others available. Note, we don’t necessarily endorse all of the conferences listed nor guarantee they meet the requirements above, but the list may help you get the ball rolling. + +Thanks to Parker Selbert, Shannon Selbert, Brian Cardarella, Alistair Woodman, and Lee Barney for feedback and helping make this a reality. \ No newline at end of file diff --git a/_posts/2025-08-05-global-elixir-meetups.markdown b/_posts/2025-08-05-global-elixir-meetups.markdown new file mode 100644 index 000000000..df098c1d6 --- /dev/null +++ b/_posts/2025-08-05-global-elixir-meetups.markdown @@ -0,0 +1,20 @@ +--- +layout: post +title: "Take part in the Global Elixir Meetups week" +authors: +- José Valim +category: Announcements +excerpt: "We are launching Global Elixir Meetups - a week where the Elixir community organizes meetups around the world to meet, learn from each other, and discuss everything related to Elixir and the Erlang VM." +--- + +> Update: Our first Global Elixir Meetups was a success with 46 meetups spread across six continents. Thanks to everyone who organized and attended. See you next time! + +We are launching [Global Elixir Meetups (GEMs)](https://globalelixirmeetups.com) - a week where the Elixir community organizes meetups around the world to meet and learn from each other. Our goal is to spark local communities, old and new, to get together and discuss everything related to Elixir and the Erlang VM. + +Our first GEM will happen on 22-28 September and, if you were looking for an opportunity to organize a meetup, now is the perfect time: visit [the Global Elixir Meetups website](https://globalelixirmeetups.com) to learn how to organize and attend. Organizers may also opt-in to live stream their meetups to the whole world directly from the website. + +![Global Elixir Meetup banner](/images/contents/gem.jpeg "Global Elixir Meetup banner") + +The Global Elixir Meetup is organized by [Software Mansion](https://swmansion.com), who brought their expertise as creators of [Membrane](https://membrane.stream) and [Elixir WebRTC](https://elixir-webrtc.org) to make it all possible. At launch, we are already counting with 7 meetups across Europe, South America, and North America, with hopefully more continents joining us soon. + +Go ahead and find your closest GEM or run your own! diff --git a/_posts/2025-08-18-interop-and-portability.markdown b/_posts/2025-08-18-interop-and-portability.markdown new file mode 100644 index 000000000..dc4895db8 --- /dev/null +++ b/_posts/2025-08-18-interop-and-portability.markdown @@ -0,0 +1,279 @@ +--- +layout: post +title: "Interoperability in 2025: beyond the Erlang VM" +authors: +- Wojtek Mach +- José Valim +category: Announcements +excerpt: "We explore the mechanisms for interoperability and portability between Elixir, other programming languages, and runtimes." +--- + +The Erlang Virtual Machine has, historically, provided three main options for interoperability with other languages and ecosystems, with different degrees of isolation: + +* [NIFs (Native Implemented Functions)](https://www.erlang.org/doc/apps/erts/erl_nif.html) integrate with third party code in the same memory space via C bindings. This translates to low overhead and best performance but it also means faulty code can bring the whole Virtual Machine down, bypassing some of Erlang's fault-tolerance guarantees + +* [Ports](https://www.erlang.org/doc/system/ports.html) start a separate Operating System process to communicate with other languages through STDIN/STDOUT, guaranteeing process isolation. In a typical Erlang fashion, ports are fully evented, concurrent, and distributed (i.e. you can pass and communicate with ports across nodes) + +* [Distributed nodes](https://www.erlang.org/doc/apps/erts/erl_dist_protocol.html) rely on Erlang well-defined distribution and serialization protocol to communicate with other runtimes. Any language can implement said protocol and act as an Erlang node, giving you full node isolation between runtimes + +Those mechanisms have led to multiple integrations between Elixir and other programming languages, such as Zig and Rust, and more recently C++, Python, and Swift, which we will explore here. + +Furthermore, alternative implementations of the Erlang VM and Elixir have brought a fourth category of **interoperability through portability**: where your Elixir program runs in a completely different environment to leverage its native capabilities, libraries, and ecosystem, while maintaining Elixir's syntax and semantics (either partially or fully). This opens up some exciting new possibilities and since this approach is still relatively uncharted territory, let's dive into it first. + +## Portability + +The [AtomVM](https://atomvm.org) is a lightweight implementation of the Erlang VM that can run on constrained environments, such as microcontrollers with just a few hundred kilobytes of memory such as ESP32, STM32 or Pico. AtomVM supports a functional subset of Erlang VM and its standard library, all optimized to run on tiny microcontrollers. + +Given its low footprint, AtomVM can also target WebAssembly, paving the way to run Elixir in web browsers and alternative WASM runtimes in the future. The [Popcorn](https://popcorn.swmansion.com) project, [recently announced at ElixirConf EU 2025](https://www.youtube.com/watch?v=ep--rQO1FRI), builds on those capabilities to provide better interoperability between Elixir and JavaScript. + +### Popcorn + +[Popcorn](https://popcorn.swmansion.com) is a library for running Elixir in web browsers, with JavaScript interoperability. Popcorn brings an extensive subset of Elixir semantics into the browser and, although it is in its infancy, it is already capable of [running interactive Elixir code entirely client side](https://popcorn.swmansion.com/demos/eval). + +And here is a quick example showing how to communicate with JavaScript from WASM: + +```elixir +defmodule HelloPopcorn do + use GenServer + + @process_name :main + + def start_link(args) do + GenServer.start_link(__MODULE__, args, name: @process_name) + end + + @impl true + def init(_init_arg) do + Popcorn.Wasm.register(@process_name) + IO.puts("Hello console!") + + Popcorn.Wasm.run_js(""" + () => { + document.body.innerHTML = "Hello from WASM!"; + } + """) + + :ignore + end +end +``` + +Popcorn could help with Elixir adoption by making it really easy to create interactive guides with executable code right there in the browser. And once it's production ready, it could enable offline, local-first applications, entirely in Elixir. + +### Hologram + +[Hologram](https://hologram.page) is a full-stack isomorphic Elixir web framework that runs on top of Phoenix. It lets developers create dynamic, interactive web applications entirely in Elixir. + +Hologram transpiles Elixir code to JavaScript and provides a complete framework including templates, components, routing, and client-server communication for building rich web applications. + +Here is a snippet of a Hologram component that handles drawing events entirely client-side, taken from the official [SVG Drawing Demo](https://hologram.page/demos/svg-drawing): + +```elixir +defmodule DrawingBoard do + use Hologram.Component + + def init(_props, component, _server) do + put_state(component, drawing?: false, path: "") + end + + def template do + ~HOLO""" + + + + """ + end + + def action(:draw_move, params, component) when component.state.drawing? do + new_path = component.state.path <> " L #{params.event.offset_x} #{params.event.offset_y}" + put_state(component, :path, new_path) + end + + def action(:start_drawing, params, component) do + new_path = component.state.path <> " M #{params.event.offset_x} #{params.event.offset_y}" + put_state(component, drawing?: true, path: new_path) + end +end +``` + +While Popcorn runs on a lightweight implementation of the Erlang VM with all of its primitives, Hologram works directly on the Elixir syntax tree. They explore distinct paths for bringing Elixir to the browser and are both in active development. + +## Native Implemented Functions (NIFs) + +NIFs allow us to write performance-critical or system-level code and call it directly from Erlang and Elixir as if it were a regular function. + +NIFs solve practical problems like improving performance or using all Operating System capabilities. NIFs run in the same Operating System process as the VM, the same memory space. With them we can use third-party native libraries, execute syscalls, interface with the hardware, etc. On the other hand, using them can forgo some of Erlang's stability and error handling guarantees. + +Originally, NIFs could never block and had to be written in a "yielding" fashion, which limited their applicability. Since Erlang/OTP 17, however, NIFs can be scheduled to run on separate OS threads called "dirty schedulers", based on their workloads (IO or CPU). This has directly brought Elixir and the Erlang VM into new domains, such as [Numerical Elixir](https://github.com/elixir-nx), and to interop with new languages and ecosystems. + +### C + +Erlang's NIFs directly target the C programming language and is used to implement low-level functionality present in Erlang's standard library: + +```c +#include + +static ERL_NIF_TERM add_int64_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int64_t a, b; + if (!enif_get_int64(env, argv[0], &a) || !enif_get_int64(env, argv[1], &b)) { + return enif_make_badarg(env); + } + return enif_make_int64(env, a + b); +} + +static ErlNifFunc nif_funcs[] = { + {"add", 2, add_int64_nif}, +}; + +ERL_NIF_INIT("Elixir.Example", nif_funcs, NULL, NULL, NULL, NULL) +``` + +Writing NIFs in C can be verbose and error-prone. Fortunately, the Elixir ecosystem offers a number of high-quality libraries that make it possible to write NIFs in other languages, let's check them out. + +### C++ + +[Fine](https://github.com/elixir-nx/fine) is a lightweight C++ library that wraps the NIF API with a modern interface. Given the widespread use of C++ in machine learning and data, Fine aims to reduce the friction of getting from Elixir to C++ and vice-versa. + +Here's the same NIF that adds two numbers in C++, using Fine: + +```c++ +#include + +int64_t add(ErlNifEnv *env, int64_t a, int64_t b) { + return a + b; +} + +FINE_NIF(add, 0); +FINE_INIT("Elixir.Example"); +``` + +Fine automatically encodes and decodes NIF arguments and return values based on the function signature, significantly reducing boilerplate code. It also has first-class support for Elixir structs, propagating C++ exceptions as Elixir exceptions, and more. + +### Rust + +[Rustler](https://github.com/rusterlium/rustler) is a library for writing NIFs in Rust. The goal is to make it impossible to crash the VM when using "safe" Rust code. Furthermore, Rustler makes it easy to encode/decode Rust values to and from Elixir terms while safely and ergonomically managing resources. + +Here's an example NIF implemented with Rustler: + +```rust +#[rustler::nif] +fn add(a: i64, b: i64) -> i64 { + a + b +} + +rustler::init!("Elixir.Example"); +``` + +### Zig + +[Zigler](https://hexdocs.pm/zigler) lets us write NIFs in Zig, a low-level programming language designed for maintaining robust, optimal, and reusable software. Zig removes hidden control flow, implicit memory allocation, and similar abstractions in favour of code that's explicit and predictable. + +Zigler compiles Zig code at build time and exposes it directly to Elixir, without external build scripts or glue. It tightly integrates with Elixir tooling: Zig code is formatted via `mix format` and documentation written in Zig appears in IEx via the `h` helper. + +Here's an example NIF in Zig: + +```elixir +iex> Mix.install([:zigler]) +iex> defmodule Example do + use Zig, otp_app: :zigler + + ~Z""" + pub fn add(a: i64, b: i64) i64 { + return a + b; + } + """ + end +iex> Example.add(1, 2) +3 +``` + +We can write NIFs directly in IEx sessions, scripts, Livebook notebooks, and similar! And with Zig's excellent interop with C, it's really easy to experiment with native code on the Erlang VM. + +### Python + +[Pythonx](https://github.com/livebook-dev/pythonx) runs a Python interpreter in the same OS process as your Elixir application, allowing you to evaluate Python code and conveniently convert between Python and Elixir data structures. Pythonx also integrates with the [uv](https://docs.astral.sh/uv/) package manager, automating the management of Python and its dependencies. + +One caveat is that Python's Global Interpreter Lock (GIL) prevents multiple threads from executing Python code at the same time so calling Pythonx from multiple Elixir processes does not provide concurrency we might expect and can become source of bottlenecks. However, GIL is a constraint for regular Python code only. Packages with CPU-intense functionality, such as `numpy`, have native implementation of many functions and invoking those releases the GIL (GIL is also released when waiting on I/O). + +Here's an example of using `numpy` in Elixir: + +```elixir +iex> Mix.install([{:pythonx, "~> 0.4.0"}]) +iex> Pythonx.uv_init(""" + [project] + name = "myapp" + version = "0.0.0" + requires-python = "==3.13.*" + dependencies = [ + "numpy==2.2.2" + ] + """) +iex> import Pythonx, only: :sigils +iex> x = 1 +iex> ~PY""" + import numpy as np + + a = np.int64(x) + b = np.int64(2) + a + b + """ +#Pythonx.Object< + np.int64(3) +> +``` + +[Livebook](https://livebook.dev) uses Pythonx to allow Elixir and Python code cells to co-exist in the same notebook (and in the same memory space), with low-overhead when transferring data between them. + +## Distributed nodes + +Elixir, by way of Erlang, has built-in support for distributed systems. Multiple nodes can connect over a network and communicate using message passing, with the same primitives such as `send` and `receive` used for both local and remote processes. + +Nodes become discoverable in the cluster simply by starting them with names. Once we connect to a node, we can send messages, spawn remote processes, and more. Here's an example: + +```shell +$ iex --name a@127.0.0.1 --cookie secret +$ iex --name b@127.0.0.1 --cookie secret +iex(a@127.0.0.1)> Node.connect(:"b@127.0.0.1") +iex(a@127.0.0.1)> node() +:"a@127.0.0.1" +iex(a@127.0.0.1)> :erpc.call(:"b@127.0.0.1", fn -> node() end) +:"b@127.0.0.1" +``` + +While Distributed Erlang is typically used for Erlang-Erlang communication, it can be also used for interacting with programs written in other programming languages. Erlang/OTP includes [Erl_Interface](https://www.erlang.org/doc/apps/erl_interface), a C library for writing programs that can participate in the Erlang cluster. Such programs are commonly called C nodes. + +Any language may implement these protocols from scratch or, alternatively, use `erl_interface` as its building block. For example, Erlang/OTP ships with [Jinterface](https://www.erlang.org/doc/apps/jinterface) application, a Java library that lets JVM programs act as distributed Erlang nodes. Another recent example is the [Swift Erlang Actor System](https://github.com/otp-interop/swift-erlang-actor-system), for communicating between Swift and Erlang/Elixir programs. + +## Ports + +Last but not least, ports are the basic mechanism that Elixir/Erlang uses to communicate with the outside world. Ports are the most common of interoperability across programming languages, so we will only provide two brief examples. + +In Elixir, the `Port` module offers a low-level API to start separate programs. Here's an example that runs `uname -s` to print the current operating system: + +```elixir +iex> port = Port.open({:spawn, "uname -s"}, [:binary]) +iex> flush() +{#Port<0.3>, {:data, "Darwin\n"}} +iex> send(port, {self(), :close}) +iex> flush() +{#Port<0.3>, :closed} +:ok +``` + +Most times, however, developers use `System.cmd/3` to invoke short-running programs: + +```elixir +iex> System.cmd("uname", ["-s"]) +{"Darwin\n", 0} +``` + +## Summary + +This article highlights many of the several options for interoperating with Elixir and the Erlang Virtual Machine. While it does not aim to be a complete reference, it covers integration across a range of languages, such as Rust, Zig, Python and Swift, as well as portability to different environments, including microcontrollers and web browsers. diff --git a/_posts/2025-10-16-elixir-v1-19-0-released.markdown b/_posts/2025-10-16-elixir-v1-19-0-released.markdown new file mode 100644 index 000000000..32c2ee283 --- /dev/null +++ b/_posts/2025-10-16-elixir-v1-19-0-released.markdown @@ -0,0 +1,219 @@ +--- +layout: post +title: "Elixir v1.19 released: enhanced type checking and up to 4x faster compilation for large projects" +authors: +- José Valim +category: Releases +excerpt: "Elixir v1.19 released: type checking of protocols and anonymous functions, broader type inference, improved compile times, and more" +--- + +Elixir v1.19 brings further improvements to the type system and compilation times, allowing us to find more bugs, faster. + +## Type system improvements + +This release improves the type system by adding type inference of anonymous functions and type checking of protocols. These enhancements seem simple on the surface but required us to go beyond existing literature by extending current theory and developing new techniques. We will outline the technical details in future articles. For now, let's look at what's new. + +### Type checking of protocol dispatch and implementations + +This release adds type checking when dispatching and implementing protocols. + +For example, string interpolation in Elixir uses the `String.Chars` protocol. If you pass a value that does not implement said protocol, Elixir will now emit a warning accordingly. + +Here is an example passing a range, which cannot be converted into a string, to an interpolation: + +```elixir +defmodule Example do + def my_code(first..last//step = range) do + "hello #{range}" + end +end +``` + +the above emits the following warnings: + +``` +warning: incompatible value given to string interpolation: + + data + +it has type: + + %Range{first: term(), last: term(), step: term()} + +but expected a type that implements the String.Chars protocol, it must be one of: + + dynamic( + %Date{} or %DateTime{} or %NaiveDateTime{} or %Time{} or %URI{} or %Version{} or + %Version.Requirement{} + ) or atom() or binary() or float() or integer() or list(term()) +``` + +Warnings are also emitted if you pass a data type that does not implement the `Enumerable` protocol as a generator to for-comprehensions: + +```elixir +defmodule Example do + def my_code(%Date{} = date) do + for(x <- date, do: x) + end +end +``` + +will emit: + +``` +warning: incompatible value given to for-comprehension: + + x <- date + +it has type: + + %Date{year: term(), month: term(), day: term(), calendar: term()} + +but expected a type that implements the Enumerable protocol, it must be one of: + + dynamic( + %Date.Range{} or %File.Stream{} or %GenEvent.Stream{} or %HashDict{} or %HashSet{} or + %IO.Stream{} or %MapSet{} or %Range{} or %Stream{} + ) or fun() or list(term()) or non_struct_map() +``` + +### Type checking and inference of anonymous functions + +Elixir v1.19 can now type infer and type check anonymous functions. Here is a trivial example: + +```elixir +defmodule Example do + def run do + fun = fn %{} -> :map end + fun.("hello") + end +end +``` + +The example above has an obvious typing violation, as the anonymous function expects a map but a string is given. With Elixir v1.19, the following warning is now printed: + +``` + warning: incompatible types given on function application: + + fun.("hello") + + given types: + + binary() + + but function has type: + + (dynamic(map()) -> :map) + + typing violation found at: + │ + 6 │ fun.("hello") + │ ~ + │ + └─ mod.exs:6:8: Example.run/0 +``` + +Function captures, such as `&String.to_integer/1`, will also propagate the type as of Elixir v1.19, arising more opportunity for Elixir's type system to catch bugs in our programs. + +### Acknowledgements + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/), [Starfish*](https://starfish.team/), and [Dashbit](https://dashbit.co/). + +## Faster compile times in large projects + +This release includes two compiler improvements that can lead up to 4x faster builds in large codebases. + +While Elixir has always compiled the given files in project or a dependency in parallel, the compiler would sometimes be unable to use all of the machine resources efficiently. This release addresses two common limitations, delivering performance improvements that scale with codebase size and available CPU cores. + +### Code loading bottlenecks + +Prior to this release, Elixir would load modules as soon as they were defined. However, because the Erlang part of code loading happens within a single process (the code server), this would make it a bottleneck, reducing parallelization, especially on large projects. + +This release makes it so modules are loaded lazily. This reduces the pressure on the code server and the amount of work during compilation, with reports of more than two times faster compilation for large projects. The benefits depend on the codebase size and the number of CPU cores available. + +Implementation wise, [the parallel compiler already acts as a mechanism to resolve modules during compilation](https://elixir-lang.org/blog/2012/04/24/a-peek-inside-elixir-s-parallel-compiler/), so we built on that. By making sure the compiler controls both module compilation and module loading, it can also better guarantee deterministic builds. + +There are two potential regressions with this approach. The first one happens if you spawn processes during compilation which invoke other modules defined within the same project. For example: + +```elixir +defmodule MyLib.SomeModule do + list = [...] + + Task.async_stream(list, fn item -> + MyLib.SomeOtherModule.do_something(item) + end) +end +``` + +Because the spawned process is not visible to the compiler, it won't be able to load `MyLib.SomeOtherModule`. You have two options, either use [`Kernel.ParallelCompiler.pmap/2`](https://hexdocs.pm/elixir/Kernel.ParallelCompiler.html#pmap/2) or explicitly call [`Code.ensure_compiled!(MyLib.SomeOtherModule)`](https://hexdocs.pm/elixir/Code.html#ensure_compiled!/1) before spawning the process that uses said module. + +The second one is related to `@on_load` callbacks (typically used for [NIFs](https://www.erlang.org/doc/system/nif.html)) that invoke other modules defined within the same project. For example: + +```elixir +defmodule MyLib.SomeModule do + @on_load :init + + def init do + MyLib.AnotherModule.do_something() + end + + def something_else do + ... + end +end + +MyLib.SomeModule.something_else() +``` + +The reason this fails is because `@on_load` callbacks are invoked within the code server and therefore they have limited ability to load additional modules. It is generally advisable to limit invocation of external modules during `@on_load` callbacks but, in case it is strictly necessary, you can set `@compile {:autoload, true}` in the invoked module to address this issue in a forward and backwards compatible manner. + +Both snippets above could actually lead to non-deterministic compilation failures in the past, and as a result of these changes, compiling these cases are now deterministic. + +### Parallel compilation of dependencies + +This release introduces a variable called `MIX_OS_DEPS_COMPILE_PARTITION_COUNT`, which instructs `mix deps.compile` to compile dependencies in parallel. + +While fetching dependencies and compiling individual Elixir dependencies already happened in parallel, as outlined in the previous section, there were pathological cases where performance gains would be left on the table, such as when compiling dependencies with native code or dependencies where one or two large files would take most of the compilation time. + +By setting `MIX_OS_DEPS_COMPILE_PARTITION_COUNT` to a number greater than 1, Mix will now compile multiple dependencies at the same time, using separate OS processes. Empirical testing shows that setting it to half of the number of cores on your machine is enough to maximize resource usage. The exact speed up will depend on the number of dependencies and the number of machine cores and some users reported up to 4x faster compilation times when using our release candidates. If you plan to enable it on CI or build servers, keep in mind it will most likely have a direct impact on memory usage too. + +## Erlang/OTP 28 support + +Elixir v1.19 officially supports Erlang/OTP 28.1+ and later. In order to support the new Erlang/OTP 28 representation for regular expressions, structs can now control how they are escaped into abstract syntax trees by defining a `__escape__/1` callback. + +On the other hand, the new representation for regular expressions in Erlang/OTP 28+ implies they can no longer be used as default values for struct fields. Therefore, this is not allowed: + +```elixir +defmodule Foo do + defstruct regex: ~r/foo/ +end +``` + +You can, however, still use regexes when initializing the structs themselves: + +```elixir +defmodule Foo do + defstruct [:regex] + + def new do + %Foo{regex: ~r/foo/} + end +end +``` + +## OpenChain certification + +Elixir v1.19 is also our first release following OpenChain compliance, [as previously announced](https://elixir-lang.org/blog/2025/02/26/elixir-openchain-certification/). In a nutshell: + + * Elixir releases now include a Source SBoM in CycloneDX 1.6 or later and SPDX 2.3 or later formats. + * Each release is attested along with the Source SBoM. + +These additions offer greater transparency into the components and licenses of each release, supporting more rigorous supply chain requirements. + +This work was performed by [Jonatan Männchen](https://maennchen.dev) and sponsored by the [Erlang Ecosystem Foundation](https://erlef.org). + +## Summary + +There are many other goodies in this release, such as improved option parsing, better debuggability and performance in ExUnit, the addition of `mix help Mod`, `mix help Mod.fun`, `mix help Mod.fun/arity`, and `mix help app:package` to make documentation accessible via shell for humans and agents, and much more. [See the CHANGELOG](https://hexdocs.pm/elixir/1.19/changelog.html) for the complete release notes. + +Happy coding! diff --git a/_posts/2025-12-02-lazier-bdds-for-set-theoretic-types.markdown b/_posts/2025-12-02-lazier-bdds-for-set-theoretic-types.markdown new file mode 100644 index 000000000..01a1718ec --- /dev/null +++ b/_posts/2025-12-02-lazier-bdds-for-set-theoretic-types.markdown @@ -0,0 +1,326 @@ +--- +layout: post +title: "Lazier Binary Decision Diagrams (BDDs) for set-theoretic types" +authors: +- José Valim +- Guillaume Duboc +category: Internals +excerpt: "This article explores the data structures used to represent set-theoretic types and the recent optimizations we have applied to them" +--- + +[The Elixir team and the CNRS are working on a set-theoretic type system for Elixir](https://elixir-lang.org/blog/2023/06/22/type-system-updates-research-dev/) which, simply put, is a type-system powered by unions, intersections, and negations. As part of the implementation of said type systems, we need an efficient way of representing said operations. This article discusses the existing approaches found in theory and practice, as well as the improvements we have introduced as part of [Elixir v1.19](/blog/2025/10/16/elixir-v1-19-0-released.markdown/). + +This article covers the implementation details of the type system. You don't need to understand these internals to use the type system, just as you don't need to know virtual machine bytecodes or compiler passes to use a programming language. Our goal is to document our progress and provide guidance for future maintainers and implementers. Let's get started. + +## DNFs - Disjunctive Normal Forms + +A Disjunctive Normal Form (DNF) is a standardized way of expressing logical formulas using only disjunctions (unions) of conjunctions (intersections). In the context of set-theoretic type systems, DNFs provide a canonical representation for union and intersection types, represented respectively as `or` and `and` in Elixir. + +In Elixir, we would represent those as lists of lists. Consider a type expression like `(A and B) or (C and D)`. This is already in DNF, it's a union of intersections, and it would be represented as: `[[A, B], [C, D]]`. This means performing unions between two DNFs is a simple list concatenation: + +```elixir +def union(dnf1, dnf2), do: dnf1 ++ dnf2 +``` + +However, more complex expressions like `A and (B or C)` need to be converted. Using distributive laws, this becomes `(A and B) or (A and C)`, which is now in DNF. In other words, the intersection of DNFs is a Cartesian product: + +```elixir +def intersection(dnf1, dnf2) do + for intersections1 <- dnf1, + intersections2 <- dnf2 do + intersections1 ++ intersections2 + end +end +``` + +The advantage of DNFs is their simple structure. Every type can be represented as unions of intersecting terms, making operations like checking if a type is empty simply a matter of checking if all unions have at least one intersection that is empty: + +```elixir +def empty?(dnf) do + Enum.all?(dnf, fn intersections -> + Enum.any?(intersections, &empty_component?/1) + end) +end +``` + + +On the other hand, the snippets above already help us build an intuition on the drawbacks of DNFs. + +First, we have seen how intersections are Cartesian products, which can lead to exponential blow ups when performing the intersection of unions. For example, `(A₁ or A₂) and (B₁ or B₂) and (C₁ or C₂)` leads to `(A₁ and B₁ and C₁) or (A₁ and B₁ and C₂) or (A₁ and B₂ and C₁) or ...`, with 8 distinct unions. + +Furthermore, if we implement unions as simple list concatenations, those unions can end up with duplicated entries, which exacerbates the exponential blow up when we perform intersections of these unions. This forces us to aggressively remove duplicates in unions, making it more complex and expensive than a concatenation. + +Despite their limitations, DNFs served us well and were the data structure used as part of Elixir v1.17 and v1.18. However, since Elixir v1.19 introduced type inference of anonymous functions, negations became more prevalent in the type system, making exponential growth more frequent. Let's understand why. + +## Inferring anonymous functions + +Imagine the following anonymous function: + +```elixir +fn + %{full_name: full} -> "#{full}" + %{first_name: first, last_name: last} -> "#{last}, #{first}" +end +``` + +We can say the first clause accepts any map with the key `full_name`. The second clause accepts any map with the keys `first_name` and `last_name` which DO NOT have the key `full_name` (otherwise they would have matched the first clause). Therefore, the inferred type should be: + +```elixir +$ %{full_name: String.Chars.t()} -> String.t() +$ %{first_name: String.Chars.t(), last_name: String.Chars.t()} and not + %{full_name: String.Chars.t()} -> String.t() +``` + +As you can see, in order to express this type, we need a negation (`not`). Or, more precisely, a difference since `A and not B` is the same as `A - B`. + +Implementing negations/differences in DNFs is relatively straightforward. Instead of lists of lists, we now use lists of two-element tuples, where the first element is a list of positive types, and the second is a list of negative types. For example, previously we said `(A and B) or (C and D)` would be represented as `[[A, B], [C, D]]`, now it will be represented as: + +```elixir +[{[A, B], []}, {[C, D], []}] +``` + +While `(A and not B) or C or D` is represented as: + +```elixir +[{[A], [B]}, {[C], []}, {[D], []}] +``` + +The difference between two DNFs is implemented similarly to intersections, except we now need to perform the Cartesian product over the positive and negative parts of each conjunction. And given anonymous functions have differences, inferring the types of anonymous functions are now exponentially expensive, which caused some projects to take minutes to compile. Not good! + +## BDDs - Binary Decision Diagrams + +Luckily, those exact issues are well documented in literature and are addressed by Binary Decision Diagrams (BDDs), introduced by [Alain Frisch (2004)](https://www.cduce.org/papers/frisch_phd.pdf) and later recalled and expanded by [Giuseppe Castagna (2016)](https://www.irif.fr/~gc/papers/covcon-again.pdf). + +BDDs represent set-theoretic operations as an ordered tree. This requires us to provide an order, any order, across all types. Given [all Elixir values have a total order](https://hexdocs.pm/elixir/Kernel.html#module-term-ordering), that's quite straightforward. Furthermore, by ordering it, we can detect duplicates as we introduce nodes in the tree. The tree can have three distinct node types: + +```elixir +type bdd() = :top or :bottom or {type(), constrained :: bdd(), dual :: bdd()} +``` + +`:top` represents the top type (where the intersection `type and :top` returns `type`) and `:bottom` represents the bottom type (where the intersection `type and :bottom` returns `:bottom`). Non-leaf nodes are represented via a three-element tuple, where the first element is the type (what we have been calling `A`, `B`... so far), the second element is called in literature the constrained branch, and the third element is the dual branch. + +In order to compute the actual type of a non-leaf node, we need to compute `(type() and constrained()) or (not type() and dual())` (hence the names constrained and dual). Let's see some examples. + +The type `A` is represented as `{A, :top, :bottom}`. This is because, if we compute `(A and :top) or (not A and :bottom)`, we get `A or :bottom`, which is equivalent to `A`. + +The type `not A` is represented as `{A, :bottom, :top}`, and it gives us `(A and :bottom) or (not A and :top)`, which yields `:bottom or not A`, which is equivalent to `not A`. + +The type `A and B`, assuming `A < B` according to a total order, is represented as `{A, {B, :top, :bottom}, :bottom}`. Expanding it node by node gives us: + +```elixir +(A and ((B and :top) or (not B and :bottom))) or (not A and :bottom) +(A and (B or :bottom)) or (not A and :bottom) +(A and B) or :bottom +(A and B) +``` + +While the difference `A and not B` is represented as `{A, {B, :bottom, :top}, :bottom}`, which we also expand node by node: + +```elixir +(A and ((B and :bottom) or (not B and :top))) or (not A and :bottom) +(A and (:bottom or not B)) or (not A and :bottom) +(A and not B) or :bottom +(A and not B) +``` + +Finally, the union `A or B` is implemented as `{A, :top, {B, :top, :bottom}}`. Let's expand it: + +```elixir +(A and :top) or (not A and ((B and :top) or (not B and :bottom))) +(A and :top) or (not A and (B or :bottom)) +A or (not A and B) +(A or not A) and (A or B) +:top and (A or B) +A or B +``` + +In other words, Binary Decision Diagrams allow us to represent unions, intersections, and differences efficiently, removing the exponential blow up. [Guillaume Duboc implemented them as part of Elixir v1.19](https://github.com/elixir-lang/elixir/pull/14693), addressing the bottlenecks introduced as part of the new type system features... but unfortunately BDDs introduced new slow downs. + +The issue with BDDs comes when applying unions to intersections and differences. Take the following type `(A and B) or C`. Since we need to preserve the order `A < B < C`, it would be represented as: + +```elixir +{A, {B, :top, {C, :top, :bottom}}, {C, :top, :bottom}} +``` + +which can be expanded as: + +```elixir +(A and ((B and :top) or (not B and ((C and :top) or (not C and :bottom))))) or (not A and ((C and :top) or (not C and :bottom))) +(A and (B or (not B and C))) or (not A and C) +(A and (B or C)) or (not A and C) +(A and B) or (A and C) or (not A and C) +(A and B) or C +``` + +As you can see, although the representation is correct, its expansion ends-up generating too many disjunctions. And while we can simplify them back to `(A and B) or C` symbolically, doing such simplications in practice are too expensive. + +In other words, the BDD expansion grows exponentially in size on consecutive unions, which is particularly troublesome because we must expand the BDD every time we check for emptiness or subtyping. + +At the end of the day, it seems we traded faster intersections/differences for slower unions. Perhaps we can have our cake and eat it too? + +## BDDs with lazy unions (or ternary decision diagrams) + +Luckily, the issue above was also forecast by Alain Frisch (2004), where he suggests an additional representation, called BDDs with lazy unions. + +In a nutshell, we introduce a new element, called `uncertain`, to each non-leaf node to represent unions: + +```elixir +type lazy_bdd() = :top or :bottom or + {type(), constrained :: lazy_bdd(), uncertain :: lazy_bdd(), dual :: lazy_bdd()} +``` + +We'll refer to the `uncertain` as unions going forward. + +The type of each non-leaf node can be computed by `(type() and constrained()) or uncertain() or (not type() and dual())`. Here are some examples: + +```elixir +A = {A, :top, :bottom, :bottom} +A and B = {A, {B, :top, :bottom, :bottom}, :bottom, :bottom} +A or B = {A, :top, {B, :top, :bottom, :bottom}, :bottom} +``` + +And, going back to `(A and B) or C`, it can be represented as: + +```elixir +{A, {B, :top, :bottom, :bottom}, {C, :top, :bottom, :bottom}, :bottom} +``` + +The duplication of `C` is fully removed. With our new representation in hand, the next step is to implement union, intersection, and difference of lazy BDDs, using the formulas found in literature and described below. + +Assuming that a lazy BDD `B` is represented as `{a, C, U, D}`, and therefore `B1 = {a1, C1, U1, D2}` and `B2 = {a2, C2, U2, D2}`, the union of the lazy BDDs `B1 or B2` can be computed as: + +```elixir +{a1, C1 or C2, U1 or U2, D1 or D2} when a1 == a2 +{a1, C1, U1 or B2, D1} when a1 < a2 +{a2, C2, B1 or U2, D2} when a1 > a2 +``` + +The intersection `B1 and B2` is: + +```elixir +{a1, (C1 or U1) and (C2 or U2), :bottom, (D1 or U1) and (D2 or U2)} when a1 == a2 +{a1, C1 and B2, U1 and B2, D1 and B2} when a1 < a2 +{a2, B1 and C2, B1 and U2, B1 and D2} when a1 > a2 +``` + +The difference `B1 and not B2` is: + +```elixir +{a1, (C1 or U1) and not (C2 or U2), :bottom, (D1 or U1) and not (D2 or U2)} when a1 == a2 +{a1, (C1 or U1) and not B2, :bottom, (D1 or U1) and not B2} when a1 < a2 +{a2, B1 and not (C2 or U2), :bottom, B1 and not (D2 or U2)} when a1 > a2 +``` + +[Guillaume Duboc first implemented lazy BDDs to represent our function types](https://github.com/elixir-lang/elixir/pull/14799), addressing some of the bottlenecks introduced alongside BDDs. Afterwards, we attempted to convert all types to use lazy BDDs, hoping they would address the remaining bottlenecks, but that was not the case. There were still some projects that type checked instantaneously in Elixir v1.18 (which used DNFs) but took minutes on v1.19 release candidates, which could only point to large unions still being the root cause. However, weren't lazy BDDs meant to address the issue with unions? + +That was the question ringing in Guillaume's head and in mine after an hours-long conversation, when we decided to call it a day. Unbeknownst to each other, we both continued working on the problem that night and the following morning. Separately, we were both able to spot the issue and converge on the same solution. + +## Lazier BDDs (for intersections) + +If you carefully look at the formulas above, you can see that intersections and differences of equal nodes cause a distribution of unions. Here is the intersection: + +```elixir +{a1, (C1 or U1) and (C2 or U2), :bottom, (D1 or U1) and (D2 or U2)} when a1 == a2 +``` + +Notice how U1 and U2 now appear on both constrained and dual parts and the whole union part of the node disappeared, now listed simply as `:bottom`. + +In addition, considering the common case where `C1 = C2 = :top` and `D1 = D2 = :bottom`, the node above becomes `{a1, :top, :bottom, U1 and U2}`, which effectively moves the unions to the dual part. If you play close attention to it, since the uncertain is now `:bottom`, we reverted back to the original BDD representation. Any further `union` on those nodes will behave exactly as in the non-lazy BDDs, which we know to be problematic. + +In other words, certain operations on lazy BDDs cause unions to revert to the previous BDD representation. So it seems lazy BDDs are not lazy enough? Could we stop this from happening? + +Guillaume and I arrived at a new formula using different approaches. Given Guillaume's approach can also be used to optimize differences, that's the one I will show below. In particular, we know the intersection of equal nodes is implemented as: + +```elixir +{a1, (C1 or U1) and (C2 or U2), :bottom, (D1 or U1) and (D2 or U2)} when a1 == a2 +``` + +If we distribute the intersection in the constrained part, we get: + +```elixir +(C1 and C2) or (C1 and U2) or (U1 and C2) or (U1 and U2) +``` + +If we distribute the intersection in the dual part, we get: + +```elixir +(D1 and D2) or (D1 and U2) or (U1 and D2) or (U1 and U2) +``` + +We can clearly see both parts have `U1 and U2`, which we can then move to the union! Leaving us with: + +```elixir +{a1, + (C1 and C2) or (C1 and U2) or (U1 and C2), + (U1 and U2), + (D1 and D2) or (D1 and U2) or (U1 and D2)} when a1 == a2 +``` + +We can then factor out `C1` in the constrained and `D1` in the dual (or `C2` and `D2` respectively), resulting in: + +```elixir +{a1, + (C1 and (C2 or U2)) or (U1 and C2), + (U1 and U2), + (D1 and (D2 or U2)) or (U1 and D2)} when a1 == a2 +``` + +While this new formula requires more operations, if we consider the common case `C1 = C2 = :top` and `D1 = D2 = :bottom`, we now have `{a1, :top, U1 and U2, :bottom}`, with the unions perfectly preserved in the middle. We independently implemented this formula and noticed it addressed all remaining bottlenecks! + +## Lazier BDDs (for differences) + +The issues we outlined above for intersections are even worse for differences. Let's check the difference formula: + +```elixir +{a1, (C1 or U1) and not (C2 or U2), :bottom, (D1 or U1) and not (D2 or U2)} when a1 == a2 +{a1, (C1 or U1) and not B2, :bottom, (D1 or U1) and not B2} when a1 < a2 +{a2, B1 and not (C2 or U2), :bottom, B1 and not (D2 or U2)} when a1 > a2 +``` + +As you can see, all operations shuffle the union nodes and return `:bottom`. But this time, we know how to improve it! Let's start with `a1 == a2`. If we expand the difference in the constrained part, we get: + +```elixir +(C1 and not C2 and not U2) or (U1 and not C2 and not U2) +``` + +If we do the same in the dual part, we have: + +```elixir +(D1 and not D2 and not U2) or (U1 and not D2 and not U2) +``` + +Unfortunately, there are no shared union terms between the constrained and dual parts, unless C2 and D2 are `:bottom`. Therefore, instead of fully rewriting the difference of equal nodes, we add the following special case: + +```elixir +{a1, C1 and not U2, U1 and not U2, D1 and not U2} +when a1 == a2 and C2 == :bottom and D2 == :bottom +``` + +We can apply a similar optimization when `a1 < a2`. The current formula: + +```elixir +{a1, (C1 or U1) and not B2, :bottom, (D1 or U1) and not B2} when a1 < a2 +``` + +The constrained part can be written as `(C1 and not B2) or (U1 and not B2)` and the dual part as `(D1 and not B2) or (U1 and not B2)`. Given `(U1 and not B2)` is shared on both parts, we can also convert it to a union, resulting in: + +```elixir +{a1, C1 and not B2, U1 and not B2, D1 and not B2} when a1 < a2 +``` + +Unfortunately, we can't apply this for `a2 > a1`, as differences are asymmetric and do not distribute over unions on the right side. Therefore, the updated formula for difference is: + +```elixir +{a1, C1 and not U2, U1 and not U2, D1 and not U2} when a1 == a2 and C2 == :bottom and D2 == :bottom +{a1, (C1 or U1) and not (C2 or U2), :bottom, (D1 or U1) and not (D2 or U2)} when a1 == a2 +{a1, C1 and not B2, U1 and not B2, D1 and not B2} when a1 < a2 +{a2, B1 and not (C2 or U2), :bottom, B1 and not (D2 or U2)} when a1 > a2 +``` + +With these new formulas, all new typing features in Elixir v1.19 perform efficiently and most projects now type check faster than in Elixir v1.18. We have also been able to use the rules above to derive additional optimizations for differences, such as when `a1 == a2 and U2 == :bottom`, which will be part of future releases. Hooray! + +## Acknowledgements + +As there is an increasing interest in implementing set-theoretic types for other dynamic languages, we hope this article shines a brief light on the journey and advancements made by the research and Elixir teams when it comes to representing set-theoretic types. + +The type system was made possible thanks to a partnership between [CNRS](https://www.cnrs.fr/) and [Remote](https://remote.com/). The development work is currently sponsored by [Fresha](https://www.fresha.com/) and [Tidewave](https://tidewave.ai/). \ No newline at end of file diff --git a/atom.xml b/atom.xml index 59ed4bffc..052b63a8e 100644 --- a/atom.xml +++ b/atom.xml @@ -1,10 +1,10 @@ --- -layout: nil +layout: null elixir_url: http://elixir-lang.org --- - + Elixir Lang @@ -15,15 +15,15 @@ elixir_url: http://elixir-lang.org {{ post.title }} - {% if post.author %} - - {{ post.author }} - - {% endif %} + {% for author in post.authors %} + + {{ author }} + + {% endfor %} {{ post.date | date_to_xmlschema }} {{ post.id }} {{ post.content | xml_escape }} {% endfor %} - + diff --git a/blog/categories.html b/blog/categories.html index f594ff33d..53589af04 100644 --- a/blog/categories.html +++ b/blog/categories.html @@ -7,8 +7,10 @@

Posts by category:

    - {% for category in site.categories %} -
    » {{ category | first | capitalize }}
    + {% assign sorted_cats = site.categories | sort %} + {% for category in sorted_cats %} +
  • +
    » {{ category | first | capitalize }}
      {% for posts in category %} {% for post in posts %} diff --git a/blog/index.html b/blog/index.html index 208c10bca..75d152e14 100644 --- a/blog/index.html +++ b/blog/index.html @@ -9,11 +9,11 @@

      {{ post.title }}

      -
      {{ post.date | date: "%B %d, %Y" }}{% if post.author %} · by {{ post.author}}{% endif %} . in {% for category in post.categories %}{{ category }}{% endfor %}
      +
      {{ post.date | date: "%B %d, %Y" }}{% if post.authors %} · by {{ post.authors | join: ", " }}{% endif %} · in {% for category in post.categories %}{{ category }}{% endfor %}
      - {{ post.excerpt }} +

      {{ post.excerpt }}

      Continue reading →

      diff --git a/cases.html b/cases.html new file mode 100644 index 000000000..42d49e97a --- /dev/null +++ b/cases.html @@ -0,0 +1,47 @@ +--- +section: cases +layout: default +image: /images/social/elixir-og-card.jpg +--- + +

      Cases

      + +

      Click on the cases below to learn more about how companies across different industries are using the power of Elixir and its ecosystem to create and grow their businesses. Cases are listed in the order they have been published.

      + +
      + {% assign sorted_posts = site.categories["Elixir in Production"] | sort: 'date' | reverse %} + {% for post in sorted_posts %} + {%- if post.flagship %} + + {% if post.logo == nil %} +
      + {% else %} +
      + {% endif %} +
      + {% for tag in post.tags %} + #{{tag}} + {% endfor %} +
      +       + {%- endif -%} + {% endfor %} + {% for post in sorted_posts %} + {%- unless post.flagship %} + + {% if post.logo == nil %} +
      + {% else %} +
      + {% endif %} +
      + {% for tag in post.tags %} + #{{tag}} + {% endfor %} +
      +       + {%- endunless -%} + {% endfor %} +
      + +

      diff --git a/crash-course.markdown b/crash-course.markdown index efe81cf65..acbb6da09 100644 --- a/crash-course.markdown +++ b/crash-course.markdown @@ -2,32 +2,23 @@ title: "Erlang/Elixir Syntax: A Crash Course" section: home layout: default +image: /images/social/elixir-og-card.jpg --- # {{ page.title }} This is a quick introduction to the Elixir syntax for Erlang developers and vice-versa. It is the absolute minimum amount of knowledge you need in order to understand Elixir/Erlang code, support interoperability, read the docs, sample code, etc. -This page is divided into sections: +{% include toc.html %} -1. [Running Code](#running_code) -2. [Notable Differences](#notable_differences) -3. [Data Types](#data_types) -4. [Modules](#modules) -5. [Function Syntax](#function_syntax) -6. [Control Flow](#control_flow) -7. [Adding Elixir to existing Erlang programs](#interop) -8. [Further reading](#further_reading) - -
      - -## 1 Running Code +## Running code ### Erlang The fastest way to run some code is to launch the Erlang shell -- `erl`. Many code snippets on this page can be pasted directly into the shell. However, when you want to define a named function, Erlang expects it to be inside of a module, and modules have to be compiled. Here's a skeleton for a module: ```erlang +% module_name.erl -module(module_name). % you may use some other name -compile(export_all). @@ -37,8 +28,8 @@ hello() -> Add your functions to it, save it to disk, run `erl` from the same directory and execute the `compile` command: -```erlang -Eshell V5.9 (abort with ^G) +```erl +Eshell V13.0.4 (abort with ^G) 1> c(module_name). ok 1> module_name:hello(). @@ -56,57 +47,64 @@ Elixir too has an interactive shell called `iex`. Compiling Elixir code can be d # module_name.ex defmodule ModuleName do def hello do - IO.puts "Hello World" + IO.puts("Hello world!") end end ``` And compiled from `iex`: -```iex -Interactive Elixir +```elixir +Interactive Elixir (1.14.0) - press Ctrl+C to exit (type h() ENTER for help) iex> c("module_name.ex") [ModuleName] -iex> ModuleName.hello +iex> ModuleName.hello() Hello world! :ok ``` -However notice that in Elixir you don't need to create a file only to create a new module, Elixir modules can be defined directly in the shell: +However, notice that in Elixir you don't need to create a file only to create a new module; Elixir modules can be defined directly in the shell: ```elixir -defmodule MyModule do - def hello do - IO.puts "Another Hello" - end -end +iex> defmodule MyModule do +...> def hello do +...> IO.puts("Another Hello") +...> end +...> end +{:module, MyModule, + <<70, 79, 82, 49, 0, 0, 5, 136, 66, 69, 65, 77, 65, 116, 85, 56, 0, 0, 0, 187, + 0, 0, 0, 19, 15, 69, 108, 105, 120, 105, 114, 46, 77, 121, 77, 111, 100, 117, + 108, 101, 8, 95, 95, 105, 110, 102, 111, ...>>, {:hello, 0}} +iex> MyModule.hello() +Another Hello +:ok ``` -
      -## 2 Notable Differences +## Notable differences This section goes over some of the syntactic differences between the two languages. -### Operator Names +### Operator names -Some operators are spelled differently. +Some operators are spelled differently, others are not available. - | Erlang | Elixir | Meaning | - ----------------------------------------------------------------------------- - | and | NOT AVAILABLE | Logical 'and', evaluates both arguments | - | andalso | and | Logical 'and', short-circuits | - | or | NOT AVAILABLE | Logical 'or', evaluates both arguments | - | orelse | or | Logical 'or', short-circuits | - | =:= | === | A match operator | - | =/= | !== | A negative match | - | /= | != | Not equals | - | =< | <= | Less than or equals | +| Erlang | Elixir | Meaning | +|----------------|----------------|--------------------------------------------------| +| and | NOT AVAILABLE | Strictly boolean 'and', evaluates both arguments | +| andalso | and | Strictly boolean "and", short-circuits | +| or | NOT AVAILABLE | Strictly boolean 'or', evaluates both arguments | +| orelse | or | Strictly boolean "or", short-circuits | +| xor | NOT AVAILABLE | Strictly boolean 'xor' | +| =:= | === | Strictly equals to | +| =/= | !== | Strictly not equals to | +| /= | != | Not equals to | +| =< | <= | Less than or equals to | ### Delimiters -Erlang expressions are terminated with a dot `.` and comma `,` is used to evaluates multiple expressions within one context (in a function definition, for instance). In Elixir, expressions are delimited by a line break or a semicolon `;`. +Erlang expressions are terminated with a dot `.` and comma `,` is used to evaluate multiple expressions within one context (in a function definition, for instance). In Elixir, expressions are delimited by a line break or a semicolon `;`. **Erlang** @@ -122,7 +120,7 @@ x = 2; y = 3 x + y ``` -### Variable Names +### Variable names Variables in Erlang can only be assigned once. The Erlang shell provides a special command `f` that allows you to erase the binding of a variable or all variables at once. @@ -130,7 +128,7 @@ Elixir allows you to assign to a variable more than once. If you want to match a **Erlang** -```erlang +```erl Eshell V5.9 (abort with ^G) 1> X = 10. 10 @@ -161,44 +159,36 @@ iex> ^a = 3 ** (MatchError) no match of right hand side value: 3 ``` -### Calling Functions - -Elixir allows you to omit parentheses in function calls, Erlang does not. - - | Erlang | Elixir | - -------------------------------------- - | some_function(). | some_function | - | sum(A, B) | sum a, b | +### Calling functions Invoking a function from a module uses different syntax. In Erlang, you would write ```erlang -orddict:new(). +lists:last([1, 2]). ``` -to invoke the `new` function from the `orddict` module. In Elixir, use the dot `.` in place of the colon `:` +to invoke the `last` function from the `lists` module. In Elixir, use the dot `.` in place of the colon `:` ```elixir -Kernel.self +List.last([1, 2]) ``` **Note**. Since Erlang modules are represented by atoms, you may invoke Erlang functions in Elixir as follows: ```elixir -:lists.sort [3, 2, 1] +:lists.sort([3, 2, 1]) ``` All of the Erlang built-ins reside in the `:erlang` module. -
      -## 3 Data Types +## Data types Erlang and Elixir have the same data types for the most part, but there are a number of differences. ### Atoms -In Erlang, an `atom` is any identifier that starts with a lowercase letter, e.g. `ok`, `tuple`, `donut`. Identifiers that start with a capital letters are always treated as variable names. Elixir, on the other hand, uses the former for naming variables, and the latter are treated as atom aliases. Atoms in Elixir always start with a colon `:`. +In Erlang, an `atom` is any identifier that starts with a lowercase letter, e.g. `ok`, `tuple`, `donut`. Identifiers that start with a capital letter are always treated as variable names. Elixir, on the other hand, uses the former for naming variables, and the latter are treated as atom aliases. Atoms in Elixir always start with a colon `:`. **Erlang** @@ -236,9 +226,11 @@ is_atom(''). %=> true **Elixir** ```elixir -is_atom :ok #=> true -is_atom :'ok' #=> true -is_atom :"Multiple words" #=> true +is_atom(:ok) #=> true +is_atom(:'ok') #=> true +is_atom(Ok) #=> true +is_atom(:"Multiple words") #=> true +is_atom(:"") #=> true ``` ### Tuples @@ -253,18 +245,18 @@ That said, Elixir does not import the default `element` and `setelement` functio **Erlang** ```erlang -element(1, { a, b, c }) %=> a -setelement(1, { a, b, c }, d) %=> { d, b, c } +element(1, {a, b, c}). %=> a +setelement(1, {a, b, c}, d). %=> {d, b, c} ``` **Elixir** ```elixir -elem({ :a, :b, :c }, 0) #=> :a -put_elem({ :a, :b, :c }, 0, :d) #=> { :d, :b, :c } +elem({:a, :b, :c}, 0) #=> :a +put_elem({:a, :b, :c}, 0, :d) #=> {:d, :b, :c} ``` -### Lists and Binaries +### Lists and binaries Elixir has a shortcut syntax for binaries: @@ -279,22 +271,22 @@ is_binary(<<"Hello">>). %=> true **Elixir** ```elixir -is_list 'Hello' #=> true -is_binary "Hello" #=> true -is_binary <<"Hello">> #=> true +is_list('Hello') #=> true +is_binary("Hello") #=> true +is_binary(<<"Hello">>) #=> true <<"Hello">> === "Hello" #=> true ``` -In Elixir, the word **string** means a utf-8 binary and there is a `String` module that works on such data. Elixir also expects your source files to be utf-8 encoded. On the other hand, **string** in Erlang refers to char lists and there is a `:string` module, that's not utf-8 aware and works mostly with char lists. +In Elixir, the word **string** means a UTF-8 binary and there is a `String` module that works on such data. Elixir also expects your source files to be UTF-8 encoded. On the other hand, **string** in Erlang refers to char lists and there is a `:string` module that works mostly with both char lists and UTF-8 encoded binaries. -Elixir also supports multiline strings (also called heredocs): +Elixir also supports multiline strings (also called *heredocs*): ```elixir -is_binary """ +is_binary(""" This is a binary spanning several lines. -""" +""") #=> true ``` @@ -305,27 +297,31 @@ Elixir offers a literal syntax for creating a list of two-item tuples where the **Erlang** ```erlang -[{another_key,20},{key,10}] +Proplist = [{another_key, 20}, {key, 10}]. +proplists:get_value(another_key, Proplist). +%=> 20 ``` **Elixir** ```elixir kw = [another_key: 20, key: 10] -kw[:another_key] #=> 20 +kw[:another_key] +#=> 20 ``` ### Maps -Erlang R17 introduced maps, a key-value store, with no ordering. Keys and values can be any term. Creating, updating and matching maps in both languages is shown below: +Maps are key-value pairs with no ordering. Keys and values can be any term. Creating, updating and matching maps in both languages is shown below: **Erlang** ```erlang -Map = #{key => 0} -Updated = Map#{key := 1} -#{key := Value} = Updated -Value =:= 1 +Map = #{key => 0}. +Updated = Map#{key := 1}. +#{key := Value} = Updated. +Value =:= 1. +%=> true ``` **Elixir** @@ -335,6 +331,7 @@ map = %{:key => 0} map = %{map | :key => 1} %{:key => value} = map value === 1 +#=> true ``` If the keys are all atoms, Elixir allows developers to use `key: 0` for defining the map as well as using `.key` for accessing fields: @@ -360,23 +357,11 @@ re:run("abc ", Pattern). **Elixir** ```elixir -Regex.run ~r/abc\s/, "abc " +Regex.run(~r/abc\s/, "abc ") #=> ["abc "] ``` -Regexes are also supported in heredocs, which is convenient when defining multiline regexes: - -```elixir -is_regex ~r""" -This is a regex -spanning several -lines. -""" -``` - -
      - -## 4 Modules +## Modules Each Erlang module lives in its own file which has the following structure: @@ -405,12 +390,12 @@ An Elixir equivalent to the Erlang above: defmodule HelloModule do # A "Hello world" function def some_fun do - IO.puts "Hello world!" + IO.puts("Hello world!") end # This one works only with lists def some_fun(list) when is_list(list) do - IO.inspect list + IO.inspect(list) end # A private function @@ -426,7 +411,7 @@ In Elixir, it is also possible to have multiple modules in one file, as well as defmodule HelloModule do defmodule Utils do def util do - IO.puts "Utilize" + IO.puts("Utilize") end defp priv do @@ -442,34 +427,33 @@ end defmodule ByeModule do end -HelloModule.dummy +HelloModule.dummy() #=> :ok -HelloModule.Utils.util -#=> "Utilize" +HelloModule.Utils.util() +# "Utilize" +#=> :ok -HelloModule.Utils.priv +HelloModule.Utils.priv() #=> ** (UndefinedFunctionError) undefined function: HelloModule.Utils.priv/0 ``` -
      - -## 5 Function Syntax +## Function syntax -[This chapter][3] from the Erlang book provides a detailed description of pattern matching and function syntax in Erlang. Here, I'm briefly covering the main points and provide sample code both in Erlang and Elixir. +[This chapter][3] from the Erlang book provides a detailed description of pattern matching and function syntax in Erlang. Here, we briefly cover the main points and provide sample code both in Erlang and Elixir. [3]: http://learnyousomeerlang.com/syntax-in-functions -### Pattern Matching +### Pattern matching Pattern matching in Elixir is based on Erlang's implementation and in general is very similar: **Erlang** ```erlang -loop_through([H|T]) -> - io:format '~p~n', [H], - loop_through(T); +loop_through([Head | Tail]) -> + io:format('~p~n', [Head]), + loop_through(Tail); loop_through([]) -> ok. @@ -478,9 +462,9 @@ loop_through([]) -> **Elixir** ```elixir -def loop_through([h|t]) do - IO.inspect h - loop_through t +def loop_through([head | tail]) do + IO.inspect(head) + loop_through(tail) end def loop_through([]) do @@ -499,9 +483,9 @@ In both Erlang and Elixir, a function is not identified only by its name, but by **Erlang** ```erlang -sum() -> 0; -sum(A) -> A; -sum(A, B) -> A + B; +sum() -> 0. +sum(A) -> A. +sum(A, B) -> A + B. sum(A, B, C) -> A + B + C. ``` @@ -532,10 +516,10 @@ sum(1, 2). %=> 3 sum([1], [2]). -%=> [1,2] +%=> [1, 2] -sum("a", "b"). -%=> "ab" +sum(<<"a">>, <<"b">>). +%=> <<"ab">> ``` **Elixir** @@ -553,16 +537,18 @@ def sum(a, b) when is_binary(a) and is_binary(b) do a <> b end -sum 1, 2 +sum(1, 2) #=> 3 -sum [1], [2] -#=> [1,2] +sum([1], [2]) +#=> [1, 2] -sum "a", "b" +sum("a", "b") #=> "ab" ``` +### Default values + In addition, Elixir allows for default values for arguments, whereas Erlang does not. ```elixir @@ -570,11 +556,11 @@ def mul_by(x, n \\ 2) do x * n end -mul_by 4, 3 #=> 12 -mul_by 4 #=> 8 +mul_by(4, 3) #=> 12 +mul_by(4) #=> 8 ``` -### Anonymous Functions +### Anonymous functions Anonymous functions are defined in the following way: @@ -593,12 +579,12 @@ lists:map(Square, [1, 2, 3, 4]). **Elixir** ```elixir -sum = fn(a, b) -> a + b end +sum = fn a, b -> a + b end sum.(4, 3) #=> 7 -square = fn(x) -> x * x end -Enum.map [1, 2, 3, 4], square +square = fn x -> x * x end +Enum.map([1, 2, 3, 4], square) #=> [1, 4, 9, 16] ``` @@ -617,40 +603,48 @@ F([]). %=> "Empty" F({a, b}). -%=> "All your {a,b} are belong to us" +%=> "All your {a, b} are belong to us" ``` **Elixir** ```elixir f = fn - {:a, :b} = tuple -> - IO.puts "All your #{inspect tuple} are belong to us" - [] -> - "Empty" - end + {:a, :b} = tuple -> + "All your #{inspect(tuple)} are belong to us" + + [] -> + "Empty" +end f.([]) #=> "Empty" f.({:a, :b}) -#=> "All your {:a,:b} are belong to us" +#=> "All your {:a, :b} are belong to us" ``` -### First-Class Functions + +### First-class functions Anonymous functions are first-class values, so they can be passed as arguments to other functions and also can serve as a return value. There is a special syntax to allow named functions be treated in the same manner. **Erlang** ```erlang +% math.erl -module(math). -export([square/1]). square(X) -> X * X. +``` -lists:map(fun math:square/1, [1, 2, 3]). -%=> [1, 4, 9] +```erl +Eshell V5.9 (abort with ^G) +1> c(math). +{ok,math} +2> lists:map(fun math:square/1, [1, 2, 3]). +[1,4,9] ``` **Elixir** @@ -662,23 +656,24 @@ defmodule Math do end end -Enum.map [1,2,3], &Math.square/1 +Enum.map([1, 2, 3], &Math.square/1) #=> [1, 4, 9] ``` -### Partials in Elixir + +### Partials and function captures in Elixir Elixir supports partial application of functions which can be used to define anonymous functions in a concise way: ```elixir -Enum.map [1, 2, 3, 4], &(&1 * 2) +Enum.map([1, 2, 3, 4], &(&1 * 2)) #=> [2, 4, 6, 8] -List.foldl [1, 2, 3, 4], 0, &(&1 + &2) +List.foldl([1, 2, 3, 4], 0, &(&1 + &2)) #=> 10 ``` -Partials also allow us to pass named functions as arguments. +We use the same `&` operator to capture a function, allowing us to pass named functions as arguments. ```elixir defmodule Math do @@ -687,14 +682,13 @@ defmodule Math do end end -Enum.map [1,2,3], &Math.square/1 - +Enum.map([1, 2, 3], &Math.square/1) #=> [1, 4, 9] ``` -
      +The above would be equivalent to Erlang's `fun math:square/1`. -## 6 Control Flow +## Control flow The constructs `if` and `case` are actually expressions in both Erlang and Elixir, but may be used for control flow as in imperative languages. @@ -705,9 +699,9 @@ The ``case`` construct provides control flow based purely on pattern matching. **Erlang** ```erlang -case { X, Y } of - { a, b } -> ok; - { b, c } -> good; +case {X, Y} of + {a, b} -> ok; + {b, c} -> good; Else -> Else end ``` @@ -715,9 +709,9 @@ end **Elixir** ```elixir -case { x, y } do - { :a, :b } -> :ok - { :b, :c } -> :good +case {x, y} do + {:a, :b} -> :ok + {:b, :c} -> :good other -> other end ``` @@ -752,14 +746,17 @@ Test_fun(10). **Elixir** ```elixir -test_fun = fn(x) -> +test_fun = fn x -> cond do x > 10 -> :greater_than_ten + x < 10 and x > 0 -> :less_than_ten_positive + x < 0 or x === 0 -> :zero_or_negative + true -> :exactly_ten end @@ -791,7 +788,7 @@ else end ``` -### Sending and Receiving Messages +### Sending and receiving messages The syntax for sending and receiving differs only slightly between Erlang and Elixir. @@ -800,10 +797,10 @@ The syntax for sending and receiving differs only slightly between Erlang and El ```erlang Pid = self(). -Pid ! { hello }. +Pid ! {hello}. receive - { hello } -> ok; + {hello} -> ok; Other -> Other after 10 -> timeout @@ -813,23 +810,21 @@ end. **Elixir** ```elixir -pid = Kernel.self +pid = Kernel.self() -send pid, { :hello } +send(pid, {:hello}) receive do - { :hello } -> :ok + {:hello} -> :ok other -> other after 10 -> :timeout end ``` -
      - -## 7 Adding Elixir to existing Erlang programs +## Adding Elixir to existing Erlang programs -Elixir compiles into BEAM byte code (via Erlang Abstract Format). This means that Elixir code can be called from Erlang and vice versa, without the need to write any bindings. All Elixir modules start with the `Elixir.` prefix followed by the regular Elixir name. For example, here is how to use the utf-8 aware `String` downcase from Elixir in Erlang: +Elixir compiles into BEAM byte code (via Erlang Abstract Format). This means that Elixir code can be called from Erlang and vice versa, without the need to write any bindings. All Elixir modules start with the `Elixir.` prefix followed by the regular Elixir name. For example, here is how to use the UTF-8 aware `String` downcase from Elixir in Erlang: ```erlang -module(bstring). @@ -839,35 +834,6 @@ downcase(Bin) -> 'Elixir.String':downcase(Bin). ``` -### Rebar integration - -If you are using rebar, you should be able to include Elixir git repository as a dependency: - - https://github.com/elixir-lang/elixir.git - -Elixir is structured similar to Erlang's OTP. It is divided into applications that are placed inside the `lib` directory, as seen in its [source code repository](https://github.com/elixir-lang/elixir). Since rebar does not recognize such structure, we need to explicitly add to our `rebar.config` which Elixir apps we want to use, for example: - -```erlang -{lib_dirs, [ - "deps/elixir/lib" -]}. -``` - -This should be enough to invoke Elixir functions straight from your Erlang code. If you are also going to write Elixir code, you can [install Elixir's rebar plugin for automatic compilation](https://github.com/yrashk/rebar_elixir_plugin). - -### Manual integration - -If you are not using rebar, the easiest approach to use Elixir in your existing Erlang software is to install Elixir using one of the different ways specified in the [Getting Started guide](http://elixir-lang.org/getting_started/1.html) and add the `lib` directory in your checkout to `ERL_LIBS`. - -
      - -## 8 Further Reading - -Erlang's official documentation site has a nice [collection][4] of programming examples. It can be a good exercise to translate them into Elixir. [Erlang cookbook][5] offers even more useful code examples. - -Elixir also provides a [Getting Started Guide][6] and has [documentation available online][7]. +## Further reading -[4]: http://www.erlang.org/doc/programming_examples/users_guide.html -[5]: http://schemecookbook.org/Erlang/TOC -[6]: http://elixir-lang.org/getting_started/1.html -[7]: http://elixir-lang.org/docs.html +Both programming languages have plenty of resources available to dig deeper. Check out their respective websites ([Elixir](https://elixir-lang.org) and [Erlang](https://www.erlang.org)) for more information. \ No newline at end of file diff --git a/css/style.css b/css/style.css index ccb5413c0..e4d6c8839 100644 --- a/css/style.css +++ b/css/style.css @@ -33,17 +33,16 @@ html { } body { margin: 0; - font: 0.8125em/1.692307em 'Bitter', Georgia, 'Times New Roman', Times, serif; /* 13 / 16 = 0.8125; 22 / 13 = 1.692307 */ - color: #555; - /* background: url("/images/texture.png") #6e4a7e; */ + font: 0.95em/1.692307em 'Bitter', Georgia, 'Times New Roman', Times, serif; + color: #24292e; padding: 0; height: 100%; } #container { - background-color: #FFF; + background-color: #fff; } a, a:visited { - color: #4E2A8E; + color: #4e2a8e; text-decoration: underline; } a.spec, a.spec:visited { @@ -51,10 +50,8 @@ a.spec, a.spec:visited { } a:hover { text-decoration: none; - color: #000; + color: #24292e; } -a:focus { outline: none; } -a:hover, a:active { outline: 0; } abbr { cursor: help } abbr[title] { border-bottom: 1px dotted; } acronym { @@ -65,7 +62,7 @@ acronym { cursor: help; } acronym:hover { - color: #333; + color: #24292e; background: #f5f5f5; border-bottom: 1px dotted #aaa; } @@ -108,13 +105,13 @@ hr { ins { font-family: Georgia, 'Times New Roman', Times, serif; background: #f5f5f5; - color: #000; + color: #24292e; text-decoration: none; font-style: italic; } mark { background: #f5f5f5; - color: #000; + color: #24292e; font-style: italic; font-weight: bold; } @@ -162,6 +159,7 @@ ol ol ol { list-style: lower-roman } ol ol ol ol { list-style: upper-alpha } ol ol ol ol ol { list-style: lower-alpha } ul ul, ol ol, ul ol, ol ul { margin-bottom:0 } + dl { margin: 0 0 1.692307em 5px } dt { font-weight: bold; @@ -172,12 +170,15 @@ h1, h2, h3, h4, h5, h6 { font-style: normal; font-weight: normal; margin: 0 0 15px 0; - color: #333; + color: #24292e; line-height: 1.3em; } -h1 { font-size: 2.769230769230769em; } /* 36 / 13 = 2.769230769230769 */ -h2 { font-size: 1.846153846153846em; } /* 24 / 13 = 1.846153846153846 */ -h3 { font-size: 1.538461538461538em; } /* 20 / 13 = 1.538461538461538 */ +h1 { font-size: 2.5384615384615383em; } /* 33 / 13 = 2.5384615384615383 */ +h2 { + font-size: 1.6923076923076923em; /* 22 / 13 = 1.6923076923076923 */ + margin-top: 1.3333333333333333em; /* 32 / 24 = 1.3333333333333333 */ +} +h3 { font-size: 1.461538461538461em; } /* 19 / 13 = 1.538461538461538 */ h4 { font-size: 1.384615384615385em; } /* 18 / 13 = 1.384615384615385 */ h5 { font-size: 1.230769230769231em; /* 16 / 13 = 1.230769230769231 */ @@ -191,6 +192,7 @@ img { border: 0; -ms-interpolation-mode: bicubic; vertical-align: middle; + max-width: 100%; } svg:not(:root) { overflow: hidden; } form { margin: 0; } @@ -250,7 +252,10 @@ th, td { text-align: left; } /* WordPress classes -------------------------------------------------------------- */ .hfeed h1, .hfeed h2, .hfeed h3, .hfeed h4, .hfeed h5, .hfeed h6 { font-weight: normal; } -.hcat h5 { font-weight: normal; margin-top: 25px; } +.hcat h5 { + font-weight: normal; + margin-top: 25px; +} .left, .alignleft { float: left; margin: 0 15px 5px 0; @@ -284,7 +289,7 @@ table { table caption { font-size: 0.8125em; line-height: 1.692307em; - color: #888; + color: #666; } table th { font-size: 0.8461538461538462em; @@ -298,10 +303,27 @@ table th { } td { padding: 0.8125em 2%; - color: #888; + color: #666; border-bottom: 1px solid #e7e7e7; } +#content table { + margin-left: 1.692307em; + width: auto; + width: 85%; +} +#content table caption { color: #24292e; } +#content table th { + font-weight: bold; + padding: 10px 4%; + border-bottom: 3px solid #E6DFD5; + background-color:#FFFAF3; +} +#content td { + padding: 0.8125em 4%; + border-bottom: 1px solid #E6DFD5; +} + /* Lists -------------------------------------------------------------- */ ul li, ol li { line-height: 2.1em; } @@ -317,41 +339,43 @@ dl dt { dl dd { margin: 0 0 5px 20px; padding: 0; - color: #888; + color: #666; } /* Blockquotes -------------------------------------------------------------- */ blockquote { overflow: hidden; - border-left: solid 3px #CCC; + border-left: solid 3px #ccc; font-size: 1.153846153846154em; /* 15 / 13 = 1.153846153846154 */ font-family: Georgia, 'Times New Roman', Times, serif; font-style: italic; - color: #aaa; + color: #666; margin: 0 0 20px 0; padding: 5px 0 5px 20px; } -blockquote pre:last-child { - margin-bottom: 0; +blockquote p:last-child, blockquote pre:last-child { + margin-bottom: 5px; } /* Code -------------------------------------------------------------- */ code { - padding: 0 3px; - color: #555; - background: #fff8ed; + padding: 1px 3px; + color: #24292e; + background: #fdfbf7; + border:1px solid #faefe0; } pre { padding: 15px 20px; - background: #fff8ed; + background: #fdfbf7; border: 1px solid #f6e4cc; } pre code { padding: 0; background: transparent; + border:0; } /* Forms @@ -366,9 +390,8 @@ form label { input[type="text"], input[type="password"], input[type="email"], .input-text, textarea, select { border: 1px solid #ddd; padding: 5px; - outline: none; font-size: 0.8125em; - color: #888; + color: #666; margin: 0; display: block; background: #fff; @@ -394,17 +417,19 @@ input[type="submit"]:hover { cursor: pointer } /* Images & Video -------------------------------------------------------------- */ a:hover img, #slider-nav li a:hover img { opacity: 0.85; } -#site-title a:hover img { border: none; opacity: 1; } -.hentry img, .entry-content img, .widget img, .hentry embed, .entry-content embed, .widget embed, .hentry object, .entry-content object, .widget object, .hentry video, .entry-content video, .widget video { +#site-title a:hover img { + border: none; + opacity: 1; +} +.hentry img, .entry-content img, .widget img, .hentry embed, .entry-content embed, .widget embed, .hentry object, .entry-content object, .widget object, .hentry video, .entry-content video, .widget video, iframe.video { max-width: 100%; } .hentry img, .entry-content img, .widget img { height: auto; padding: 1px; - border: 1px solid #e5e5e5; } .rss-button { margin-bottom: 25px; } -img.no-border { border: 0 }; +iframe.video { border:0; } /* Layout -------------------------------------------------------------- */ @@ -437,35 +462,33 @@ img.no-border { border: 0 }; -------------------------------------------------------------- */ #menu-primary { font-size: 1em; - width: 75%; - margin: 30px 0 12px 0; - float: right; - position: relative; - z-index: 99; } -#menu-primary .menu { float: right; margin-left: 0; } -#menu-primary .menu ul { float: right; margin: 0; } +#menu-primary ul { + margin: 0 0 10px; +} #menu-primary li { - display: block; - float: left; - min-height: 30px; + display: inline-block; } #menu-primary li a { - line-height: 1em; - font-size: 14px; - margin-left: 20px; - display: block; + line-height: 1.2em; + font-size: 1em; + margin-right: 20px; text-transform: uppercase; } +#menu-primary li:last-child a { + margin-right: 0; +} #menu-primary li.current-menu-item a { color: #aaa; } #menu-primary ul li a:hover { color: #222; } body.home div.menu li.home a, body.install div.menu li.install a, -body.getting_started div.menu li.getting_started a, +body.getting-started div.menu li.getting-started a, body.blog div.menu li.blog a, body.docs div.menu li.docs a, -body.source div.menu li.source a { +body.development div.menu li.development a, +body.cases div.menu li.cases a, +body.learning div.menu li.learning a { color: #aaa; } @@ -474,61 +497,38 @@ body.source div.menu li.source a { #header { clear: both; width: 100%; + display: flex; + flex-wrap: wrap; + justify-content: space-between; + margin: 20px 0; } #branding { - float: left; - width: 25%; - overflow: hidden; - margin-top: 20px; -} -#site-title { - font-size: 2.307692307692308em; /* 30 / 13 = 2.307692307692308 */ - line-height: 1em; - font-weight: bold; - margin: 0 0 20px 5px; - text-transform: uppercase; - letter-spacing: -2px; - font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; - word-wrap: break-word; -} -#site-title a { - color: #222; - border-bottom: none; + max-width: 225px; + margin-bottom: 25px; } -#header img { max-width: 100%; } -#header { - padding-bottom: 10px; +.sidebar .widget.news { + font-size: 0.8em; + margin-bottom: 7px; } -#site-description { - display: none; +.sidebar .widget.events { + margin-bottom: 5px; } -#site-description { - font-size: 20px; - line-height: 1.5em; - margin: 0; - color: #333; - text-transform: none; - float: left; - width: 91.48936170212766%; - clear: both; - text-align: center; - border-top: 5px solid #444; - padding: 18px 4.25531914893617% 20px 4.25531914893617%; /* 40px / 940px = 4.25531914893617% */ - border-bottom: 1px solid #e5e5e5; - margin-bottom: 30px; +.widget.events a { + display: inline-block; + padding-bottom: 21px; + text-decoration: none; } -.sidebar .widget.news { - color: #333; - font-size: 0.8em; - margin-bottom: 7px; +.widget.events a:hover { + color: #24292e; } -.widget.news h3 a { - color: #333; +.widget.events a img { + height: auto; + max-width: 240px; } /* Posts @@ -544,12 +544,6 @@ body.source div.menu li.source a { .hentry:last-child { border-bottom: 0; } -.singular .hentry { - margin: 0 0 30px 0; - position: relative; - float: left; - width: 100%; -} .featured { margin-bottom: 20px; } .sticky .sticky-header { float: left; @@ -569,28 +563,88 @@ body.source div.menu li.source a { height: 150px; margin: 3px 25px 20px 0; } -.no-border { border: 0 }; /* Post titles -------------------------------------------------------------- */ .hentry .entry-title { - margin: 0 0 0.6em 0; + margin: 0 0 0.25em 0; padding: 0; - font-size: 1.230769230769231em; /* 16 / 13 = 1.230769230769231 */ text-transform: uppercase; line-height: 1.3em; border: none; - color: #333; word-spacing: 2px; } -.singular .entry-title { - font-size: 1.846153846153846em; /* 24 / 13 = 1.846153846153846 */ - margin-bottom: 0.625em; -} -.singular-page .entry-title { color: #ccc; } -.singular-page .entry-title, .singular-attachment .entry-title { margin-bottom: 1.2em; } .entry-title a, .entry-title a:visited { color: #222; } -.entry-title a:hover { color: #4E2A5E; } +.entry-title a:hover { color: #4e2a5e; } + +/* Cases +-------------------------------------------------------------- */ +.cases-header { + display: flex; + flex-wrap: wrap; + justify-content: space-between; +} + +.cases-boxes { + display: flex; + flex-wrap: wrap; + justify-content: space-around; +} + +.cases-box { + margin: 1em 0.25em; + text-decoration: none; + width: 205px; +} + +#shuffled-cases .cases-box { + display: none; +} + +#shuffled-cases .cases-box:nth-child(-n+3) { + display: block; +} + +.cases-image { + display: flex; + justify-content: center; + width: 175px; + height: 125px; + padding: 10px; + outline-style: solid; + outline-width: 1.5px; + outline-color: #E5E5E5; + margin: auto; + background-repeat: no-repeat; + background-position: center; + background-size: 140px; +} + +.cases-tag { + text-align: center; + padding: 10px 0; + margin: 0; +} + +.cases-tag span { + display: inline-block; + font-size: 13px; + text-decoration: none; + color: #ffffff; + padding: 2px 3px; + background-color: #9DA1A6; + border-radius: 3px; + margin: 5px 3px; +} + +.cases-box:hover .cases-image { + outline-width: 2px; + outline-color: #4E2A8E; +} + +.cases-box:hover .cases-tag span { + background-color: #4E2A8E; +} /* Post bylines/datelines -------------------------------------------------------------- */ @@ -603,9 +657,8 @@ body.source div.menu li.source a { line-height: 1.692307em; word-spacing: 2px; } -.singular .byline { margin-bottom: 1.7em; } .byline a, .byline a:visited { color: #aaa; } -.byline a:hover { color: #000; } +.byline a:hover { color: #24292e; } .author, .published, .category, .edit { font-family: 'Bitter', Georgia, 'Times New Roman', Times, serif; font-style: normal; @@ -628,67 +681,32 @@ body.source div.menu li.source a { font-size: 0.8461538461538462em; color: #aaa; } -.entry-meta a { color: #888; } -.entry-meta a:hover { color: #000; } - -/* Singular post prev/next links --------------------------------------------------------------- */ -.singular .loop-nav { - font-size: 0.8461538461538462em; - color: #888; - clear: left; -} - -/* Page links for multi-paged posts --------------------------------------------------------------- */ -.page-links { - clear: both; - font-size: 0.8461538461538462em; - word-spacing: 2px; - line-height: 1em; - color: #222; -} -.entry-summary .page-links { - clear: none; - font-size: 0.8461538461538462em; - line-height: 1em; - color: #aaa; -} -.page-links a, .page-links a:visited { - display: inline-block; - color: #555; - background: #e9e9e9; - padding: 3px 6px; -} -.page-links a:hover { - color: #fff; - background: #555; -} +.entry-meta a { color: #666; } +.entry-meta a:hover { color: #24292e; } /* Archive/search pagination and comment pagination -------------------------------------------------------------- */ -.comment-navigation { margin-bottom: 1.692307em; } .pagination.loop-pagination { text-align: center; clear: both; - margin: 7px 0; + padding: 15px 0 5px; } -.pagination .page-numbers, .comment-navigation .page-numbers { +.pagination .page-numbers { display: inline-block; padding: 4px 8px; margin: 0; line-height: 1em; color: #444; } -.pagination a.page-numbers, .comment-navigation a.page-numbers { - color: #333; +.pagination a.page-numbers { + color: #24292e; background: #e9e9e9; } -.pagination a:hover, .comment-navigation a:hover { +.pagination a:hover { color: #fff; - background: #555; + background: #000; } -.pagination .current, .comment-navigation .current { color: #aaa; } +.pagination .current { color: #aaa; } /* Widgets -------------------------------------------------------------- */ @@ -696,20 +714,16 @@ body.source div.menu li.source a { float: left; width: 100%; margin-bottom: 26px; - color: #888; + color: #666; } -.widget table, .widget ul, .widget ol { margin: 0 0 0 16px; } -li.image { - list-style: none; - margin-bottom: 10px; -} +.widget table, .widget ul, .widget ol { margin: 0 0 0 16px; } /* Widget titles -------------------------------------------------------------- */ .sidebar .widget-title { font-size: 0.7692307692307692em; /* 10 / 13 = 0.7692307692307692 */ - color: #aaa; + color: #24292e; text-transform: uppercase; letter-spacing: 1px; word-spacing: 2px; @@ -721,14 +735,14 @@ li.image { .widget .search-form label { font-size: 0.8461538461538462em; line-height: 1.692307em; - color: #aaa; + color: #666; } .widget .search-form input[type="text"] { width: 91.538461%; float: left; padding: 8px 10px; font-size: 1em; - color: #aaa; + color: #666; } .search-form .search-submit { display: none; } @@ -740,85 +754,243 @@ li.image { padding: 15px 10px; border-top: 5px solid #444; clear: both; - color: #FFF; + color: #fff; } -#footer a { color: #FFF; text-decoration: underline; } - -#social { float: right; margin-top: -15px; } -#social ul { list-style: none; margin: 0; } -#social li { float: right; margin-left: -10px; } +#footer a { + color: #fff; + text-decoration: underline; +} -#copyright { +#trademark { text-align: center; font-size: 12px; padding: 0 0 10px; } +#edit-on-github { + padding: 10px 0 15px; + text-align: center; + font-size: 12px; +} + /* Media Queries (mobile browsing) ----------------------------------------------------- */ /* Tablet (portrait) */ @media only screen and (min-width: 768px) and (max-width: 959px) { - .wrap { max-width: 728px; } + .wrap { + max-width: 100%; + padding: 0 25px; + } .widget .search-form input[type="text"] { width: 89%; } .widget table { font-size: 0.8461538461538462em; } #footer .widget table { width: 90%; } } /* Mobile (portrait) */ @media only screen and (max-width: 767px) { - .wrap { max-width: 300px; } + .wrap { + max-width: 100%; + padding: 0 15px; + } body { line-height: 1.615384615384615em; } p { margin-bottom: 1.615384615384615em; } - #branding { - float: left; - width: 100%; - position: relative; + .hentry .archive-thumbnail { + width: 110px; + height: 110px; } #site-description { font-size: 1.3em; } - #menu-primary { - float: left; - clear: both; - width: 100%; - margin-top: 5px; - } - #menu-primary .menu { float: left; } - #menu-primary ul li { clear: left; } - #menu-primary ul li a { margin-left: 0; } #content { width: 100%; } .comment-list li li { padding-left: 0; } #sidebar-primary { width: 100%; clear: left; + border-top: 1px dotted #CCC; + padding-top: 45px; + } + #created-by ul { + margin: 0; } - .col-0 { width: 100% } - .col-1 { width: 100% } - .col-2 { width: 100% } - .col-3 { width: 100% } - .col-4 { width: 50% } - .col-5 { width: 50% } - .col-6 { width: 33.33% } - .col-7 { width: 33.33% } - .col-8 { width: 33.33% } - .col-9 { width: 33.33% } - .col-10 { width: 33.33% } - .col-11 { width: 33.33% } - .col-12 { width: 33.33% } - .col-13 { width: 33.33% } - .col-14 { width: 33.33% } - .col-15 { width: 33.33% } - .col-16 { width: 33.33% } - .col-17 { width: 33.33% } - .col-18 { width: 33.33% } - .col-19 { width: 33.33% } - .col-20 { width: 33.33% } - .col-21 { width: 33.33% } - .col-22 { width: 33.33% } - .col-23 { width: 33.33% } - .col-24 { width: 33.33% } - .col-25 { width: 33.33% } - .col-26 { width: 33.33% } - .col-27 { width: 33.33% } - .col-28 { width: 33.33% } - .col-29 { width: 33.33% } - .col-30 { width: 33.33% } } + +/* Elixir-lang.org - Pages and Specific Elements +------------------------------------------------------ */ + +/* Learning */ +body.learning a.cover { + float: left; + padding: 0.769230769em 1.538461538em 0.769230769em 0; +} + +body.learning h4.resource { + padding-top: 1em; +} + +body.learning .free { + background-color: #4E2A8E; + color: #ffffff; + display: inline-block; + font-size: 14px; + text-decoration: none; + padding: 0 5px; + border-radius: 5px; + line-height: 18px; + margin-left: 5px; +} + +/* Jekyll Table of Contents */ +ol.jekyll-toc, +ol.jekyll-toc ol { + counter-reset: item; +} +ol.jekyll-toc { + margin: 1.692307em 0 1.692307em 2.3076923076923075em; +} +ol.jekyll-toc ol { + margin: 0.615384615em 0 -0.7692307692307693em 0; +} +ol.jekyll-toc li { + display: block; +} +ol.jekyll-toc li:before { + content: counters(item, ".") " "; + counter-increment: item; +} +ol.jekyll-toc li { + list-style-position: inside; + text-indent: -1em; + padding-left: 1em; + line-height: 1.3em; + padding-bottom: 0.8em; +} +ol.jekyll-toc li a { + position:relative; + left: 0.3076923076923077em; +} + +/* heading */ +.jekyll-toc-header { + position: relative; + display: block; + margin-left: -20px; + padding-left: 20px; + + margin-right: 0; + padding-right: 20px; + cursor: default; +} + +/* anchor */ +.jekyll-toc-header a.jekyll-toc-anchor { + display: block; + position:absolute; + text-decoration: none; +} +.jekyll-toc-header a.jekyll-toc-anchor, +.jekyll-toc-header a.jekyll-toc-anchor:hover { + color: #24292e +} +.jekyll-toc-header:hover a.jekyll-toc-anchor { + cursor: pointer; +} +.jekyll-toc-header a.jekyll-toc-link-here { + left: 0; + top: 0; +} +.jekyll-toc-header a.jekyll-toc-back-to-top { + right: 0; + top: 0; +} + +/* span */ +.jekyll-toc-header span.jekyll-toc-icon { + display:none; + vertical-align: middle; + color: #24292e; +} +.jekyll-toc-header:hover span.jekyll-toc-icon { + display:inline-block; + width: auto; +} +.jekyll-toc-header a.jekyll-toc-back-to-top span.jekyll-toc-icon { + margin-left: 10px; + width:1em; + text-align: right; +} + +/* extending jekyll-toc */ +.jekyll-toc-header .icon { + font: normal normal normal 16px/1 icons !important; + text-decoration: none; + text-rendering: auto; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +/* mini documentary */ +#mini-docu a { + display: block; + margin-bottom: 12px; + text-decoration: none; +} + +.mini-docu-cta { + background-image: url(/images/learning/mini-docu.png); + background-repeat: no-repeat; + border: 2px solid #eee; + border-radius: 30px; + min-width: 208px; + padding: 7px 20px 4px 7px; + background-position: 15px 0px; + text-align: center; +} + +.mini-docu-copy { + color: #4e2a8e; + font-family: 'Bitter'; + height: 48px; + margin-left: 66px; +} + +.mini-docu-copy:hover { + color: black; +} + +/* Top banner */ +#top-banner { + background: #fdfbf7; + border-bottom: 1px solid #e4d9c9; + line-height: 1; + text-align: center; + padding: 21px 30px 16px 30px; + margin-bottom: 5px; + position: relative; +} + +#top-banner .close { + cursor: pointer; + position: absolute; + right: 0; + top: 0; + padding: 13px 18px; + font-size: 22px; + opacity: 0.35; +} + +#top-banner .close:hover { + opacity: 0.7; +} + +/* Getting started */ +.getting-started-title h1 { + font-size: 2.3em; +} + +.getting-started-title small { + font-size: 1em; + color: #666; + display: block; +} \ No newline at end of file diff --git a/css/syntax.css b/css/syntax.css index 1e651cf79..9dc212c54 100644 --- a/css/syntax.css +++ b/css/syntax.css @@ -1,47 +1,47 @@ -.highlight { background: #ffffff; } -.highlight .c { color: #999988; font-style: italic } /* Comment */ +.highlight {} +.highlight .c { color: #998; font-style: italic } /* Comment */ .highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ -.highlight .k { font-weight: bold } /* Keyword */ -.highlight .o { font-weight: bold } /* Operator */ -.highlight .cm { color: #999988; font-style: italic } /* Comment.Multiline */ -.highlight .cp { color: #999999; font-weight: bold } /* Comment.Preproc */ -.highlight .c1 { color: #999988; font-style: italic } /* Comment.Single */ -.highlight .cs { color: #999999; font-weight: bold; font-style: italic } /* Comment.Special */ -.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ -.highlight .gd .x { color: #000000; background-color: #ffaaaa } /* Generic.Deleted.Specific */ +.highlight .k { color: #54585b; font-weight: bold } /* Keyword */ +.highlight .o { color: #54585b; font-weight: bold } /* Operator */ +.highlight .cm { color: #998; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #999; font-weight: bold } /* Comment.Preproc */ +.highlight .c1 { color: #998; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #999; font-weight: bold; font-style: italic } /* Comment.Special */ +.highlight .gd { color: #000; background-color: #fdd } /* Generic.Deleted */ +.highlight .gd .x { color: #000; background-color: #faa } /* Generic.Deleted.Specific */ .highlight .ge { font-style: italic } /* Generic.Emph */ -.highlight .gr { color: #aa0000 } /* Generic.Error */ -.highlight .gh { color: #999999 } /* Generic.Heading */ -.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ -.highlight .gi .x { color: #000000; background-color: #aaffaa } /* Generic.Inserted.Specific */ -.highlight .go { color: #888888 } /* Generic.Output */ -.highlight .gp { color: #555555 } /* Generic.Prompt */ -.highlight .gs { font-weight: bold } /* Generic.Strong */ -.highlight .gu { color: #aaaaaa } /* Generic.Subheading */ -.highlight .gt { color: #aa0000 } /* Generic.Traceback */ -.highlight .kc { font-weight: bold } /* Keyword.Constant */ -.highlight .kd { font-weight: bold } /* Keyword.Declaration */ -.highlight .kp { font-weight: bold } /* Keyword.Pseudo */ -.highlight .kr { font-weight: bold } /* Keyword.Reserved */ -.highlight .kt { color: #445588; font-weight: bold } /* Keyword.Type */ -.highlight .m { color: #009999 } /* Literal.Number */ +.highlight .gr { color: #a00 } /* Generic.Error */ +.highlight .gh { color: #999 } /* Generic.Heading */ +.highlight .gi { color: #000; background-color: #dfd } /* Generic.Inserted */ +.highlight .gi .x { color: #000; background-color: #afa } /* Generic.Inserted.Specific */ +.highlight .go { color: #888 } /* Generic.Output */ +.highlight .gp { color: #555 } /* Generic.Prompt */ +.highlight .gs { color: #54585b; font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #aaa } /* Generic.Subheading */ +.highlight .gt { color: #a00 } /* Generic.Traceback */ +.highlight .kc { color: #54585b; font-weight: bold } /* Keyword.Constant */ +.highlight .kd { color: #54585b; font-weight: bold } /* Keyword.Declaration */ +.highlight .kp { color: #54585b; font-weight: bold } /* Keyword.Pseudo */ +.highlight .kr { color: #54585b; font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #458; font-weight: bold } /* Keyword.Type */ +.highlight .m { color: #099 } /* Literal.Number */ .highlight .s { color: #d14 } /* Literal.String */ .highlight .na { color: #008080 } /* Name.Attribute */ .highlight .nb { color: #0086B3 } /* Name.Builtin */ -.highlight .nc { color: #445588; font-weight: bold } /* Name.Class */ +.highlight .nc { color: #458; font-weight: bold } /* Name.Class */ .highlight .no { color: #008080 } /* Name.Constant */ .highlight .ni { color: #800080 } /* Name.Entity */ -.highlight .ne { color: #990000; font-weight: bold } /* Name.Exception */ -.highlight .nf { color: #990000; font-weight: bold } /* Name.Function */ -.highlight .nn { color: #555555 } /* Name.Namespace */ +.highlight .ne { color: #900; font-weight: bold } /* Name.Exception */ +.highlight .nf { color: #900; font-weight: bold } /* Name.Function */ +.highlight .nn { color: #555 } /* Name.Namespace */ .highlight .nt { color: #000080 } /* Name.Tag */ .highlight .nv { color: #008080 } /* Name.Variable */ -.highlight .ow { font-weight: bold } /* Operator.Word */ -.highlight .w { color: #bbbbbb } /* Text.Whitespace */ -.highlight .mf { color: #009999 } /* Literal.Number.Float */ -.highlight .mh { color: #009999 } /* Literal.Number.Hex */ -.highlight .mi { color: #009999 } /* Literal.Number.Integer */ -.highlight .mo { color: #009999 } /* Literal.Number.Oct */ +.highlight .ow { color: #54585b; font-weight: bold } /* Operator.Word */ +.highlight .w { color: #bbb } /* Text.Whitespace */ +.highlight .mf { color: #099 } /* Literal.Number.Float */ +.highlight .mh { color: #099 } /* Literal.Number.Hex */ +.highlight .mi { color: #099 } /* Literal.Number.Integer */ +.highlight .mo { color: #099 } /* Literal.Number.Oct */ .highlight .sb { color: #d14 } /* Literal.String.Backtick */ .highlight .sc { color: #d14 } /* Literal.String.Char */ .highlight .sd { color: #d14 } /* Literal.String.Doc */ @@ -53,8 +53,8 @@ .highlight .sr { color: #009926 } /* Literal.String.Regex */ .highlight .s1 { color: #d14 } /* Literal.String.Single */ .highlight .ss { color: #990073 } /* Literal.String.Symbol */ -.highlight .bp { color: #999999 } /* Name.Builtin.Pseudo */ +.highlight .bp { color: #999 } /* Name.Builtin.Pseudo */ .highlight .vc { color: #008080 } /* Name.Variable.Class */ .highlight .vg { color: #008080 } /* Name.Variable.Global */ .highlight .vi { color: #008080 } /* Name.Variable.Instance */ -.highlight .il { color: #009999 } /* Literal.Number.Integer.Long */ \ No newline at end of file +.highlight .il { color: #099 } /* Literal.Number.Integer.Long */ diff --git a/development.markdown b/development.markdown new file mode 100644 index 000000000..b84d2552e --- /dev/null +++ b/development.markdown @@ -0,0 +1,56 @@ +--- +title: "Development & Team" +section: development +layout: default +image: /images/social/elixir-og-card.jpg +--- + +# {{ page.title }} + +This page outlines the language's past and future development. + +## Development + +José Valim created Elixir in 2012 as a Research and Development project inside Plataformatec. Elixir's goal is to be [a productive and extensible language](/blog/2013/08/08/elixir-design-goals/) for writing maintainable and reliable software. + +Elixir runs on top of the Erlang Virtual Machine, which provides a scalable and fault-tolerant foundation. Elixir was designed to leverage this foundation without performance costs and aims to contribute to the wider ecosystem whenever possible. + +Elixir's source code is under the [Apache 2 License](https://github.com/elixir-lang/elixir/blob/main/LICENSE) and is maintained by the [Elixir Team](#team). The source code and contribution guidelines can be found on [the language repository](https://github.com/elixir-lang/elixir). + +Elixir v1.0 was released in September 2014 and a new minor version is released every 6 months, around May and November of every year. New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann) with a link to the complete CHANGELOG. All security releases [will be tagged with "[security]"](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). Security vulnerabilities should be disclosed to [elixir-security@googlegroups.com](mailto:elixir-security@googlegroups.com). Our [compatibility and deprecation policies](https://hexdocs.pm/elixir/compatibility-and-deprecations.html#content) are also documented. + +Since v1.0, the language development has become focused to provide a compact and consistent core. The Elixir team focuses on language features that: + + 1. are necessary for developing the language itself + 2. bring important concepts/features to the community in a way its effect can only be maximized or leveraged by making it part of the language + +The language development is open, both in terms of source code and of collaborations. All features and bug fixes planned for the next releases can be found [in the issues tracker](https://github.com/elixir-lang/elixir/issues). Features that may cause a larger impact on the ecosystem are first proposed to the community in [the Elixir mailing list](https://groups.google.com/group/elixir-lang-core) as well as in [the "Elixir News" section in the Elixir Forum](https://elixirforum.com/c/elixir-news). + +Community members are welcome to propose new features for Elixir. Before submitting a proposal, members are encouraged to gather feedback from around the community in whatever venues seem best. However, in order for a proposal to be considered for inclusion by the Elixir Core team, it must go through the Elixir mailing list. This often includes discussion and refinement of the proposal. The Elixir Core team has the final say on whether a proposal is accepted or rejected. While members are encouraged to gain support from the rest of the community, popularity does not mean that a proposal will be accepted. + +To remain focused, Elixir trusts its ecosystem to bring diversity and broaden its use cases. Therefore the language was designed to be extensible: the constructs available to build the language are also available for developers to extend the language and bring it to different domains. Projects such as [the Phoenix web framework](http://phoenixframework.org), [the Nerves embedded framework](http://nerves-project.org), and [Numerical Elixir](https://github.com/elixir-nx/nx) are such examples. + +Elixir also relies on a vibrant community to support its growth. The community is behind the meetups, events, learning resources, open source projects, and more. See the sidebar, the [Learning Resources](/learning.html) and [the Hex Package Manager website](https://hex.pm/) for some examples and more information. + +The best way to support the language is by getting involved in its community and contributing to the ecosystem. + +Welcome! + +## Team + +The Elixir Team is composed by: + + * José Valim + * Eric Meadows-Jönsson + * Andrea Leopardi + * Fernando Tapia Rico + * Jean Klingler + +We are also grateful to the past members of the Elixir Team: + + * Alexei Sholik + * James Fish + * Aleksei Magusev + * Michał Muskała + +Finally, we thank all of our [contributors](https://github.com/elixir-lang/elixir/graphs/contributors). diff --git a/docs.markdown b/docs.markdown index 3acce2ab1..55c20390a 100644 --- a/docs.markdown +++ b/docs.markdown @@ -2,26 +2,48 @@ title: Elixir Documentation section: docs layout: default +image: /images/social/elixir-og-card.jpg --- -## Documentation +# Documentation -Choose which version you want documentation for. +The Elixir programming language is broken into 6 applications. The links below +reference the documentation for the modules and functions in each of those +applications. See also [our Getting Started guide](https://hexdocs.pm/elixir/introduction.html) +and [the Learning page](/learning.html) for books, courses, videos, and more. -#### v1.0 +{% assign stable = site.data.elixir-versions[site.data.elixir-versions.stable] %} -* [Elixir](/docs/stable/elixir) - standard library -* [EEx](/docs/stable/eex) - templating library -* [ExUnit](/docs/stable/ex_unit) - unit test library -* [IEx](/docs/stable/iex) - interactive shell -* [Logger](/docs/stable/logger) - builtin Logger -* [Mix](/docs/stable/mix) - build tool +{% for version in site.data.elixir-versions %} + {% if version[0] == 'stable' %} + {% continue %} + {% endif %} -#### Master +

      + {{ version[1].name }} + {% if version[1].version == stable.version %}(stable){% endif %} + (download) +

      -* [Elixir](/docs/master/elixir) - standard library -* [EEx](/docs/master/eex) - templating library -* [ExUnit](/docs/master/ex_unit) - unit test library -* [IEx](/docs/master/iex) - interactive shell -* [Logger](/docs/master/logger) - builtin Logger -* [Mix](/docs/master/mix) - build tool +{% if version[1].otp_versions %} +Supported Erlang/OTP versions: {% for otp in version[1].otp_versions reversed %}{{ otp }}{% if forloop.last %}{% else %}, {% endif %}{% endfor %}. +{% endif %} + +* [Elixir](https://hexdocs.pm/elixir/{{ version[1].version }}/) - standard library +* [EEx](https://hexdocs.pm/eex/{{ version[1].version }}/) - templating library +* [ExUnit](https://hexdocs.pm/ex_unit/{{ version[1].version }}/) - unit test library +* [IEx](https://hexdocs.pm/iex/{{ version[1].version }}/) - interactive shell +* [Logger](https://hexdocs.pm/logger/{{ version[1].version }}/) - built-in Logger +* [Mix](https://hexdocs.pm/mix/{{ version[1].version }}/) - build tool + +
      +{% endfor %} + +#### Development + +* [Elixir](https://hexdocs.pm/elixir/main/) - standard library +* [EEx](https://hexdocs.pm/eex/main/) - templating library +* [ExUnit](https://hexdocs.pm/ex_unit/main/) - unit test library +* [IEx](https://hexdocs.pm/iex/main/) - interactive shell +* [Logger](https://hexdocs.pm/logger/main/) - built-in Logger +* [Mix](https://hexdocs.pm/mix/main/) - build tool diff --git a/downloads/cheatsheets/README.md b/downloads/cheatsheets/README.md new file mode 100644 index 000000000..1fb6e673e --- /dev/null +++ b/downloads/cheatsheets/README.md @@ -0,0 +1,21 @@ +This directory contains TeX source files and their corresponding +PDF files, containing small help pages, reminders, compressed +references, also called cheat sheets. + +To produce a .pdf starting from the .tex source, you need `tex`, +and more specifically [LaTeX](https://www.latex-project.org/get/). + +The BasicTex variant can be used as long as the following packages +are available: + + $ tlmgr install courier framed charter enumitem ec helvetica + +To compile a tex file EQUIS.tex into its corresponding EQUIS.pdf: + + $ texi2pdf EQUIS.tex + +You may ignore all output except the very last two lines, which +should look like: + + Output written on EQUIS.pdf (2 pages, 69846 bytes). + Transcript written on EQUIS.log. diff --git a/downloads/cheatsheets/gen-server.pdf b/downloads/cheatsheets/gen-server.pdf new file mode 100644 index 000000000..e67c6125f Binary files /dev/null and b/downloads/cheatsheets/gen-server.pdf differ diff --git a/downloads/cheatsheets/gen-server.tex b/downloads/cheatsheets/gen-server.tex new file mode 100644 index 000000000..5465462d1 --- /dev/null +++ b/downloads/cheatsheets/gen-server.tex @@ -0,0 +1,403 @@ +\documentclass[a4paper]{article} +\usepackage{framed} +\usepackage{tikz} +\usepackage{times,charter} +\usepackage{amssymb} +\usepackage{verbatim} +\usepackage[bindingoffset=0in,% + left=1.3cm,right=1.3cm,top=1.5cm,bottom=0.8cm,% + footskip=0in]{geometry} +\usetikzlibrary{shapes,arrows,calc,backgrounds,shapes.geometric} +% Define a background layer, in which the parchment shape is drawn +\pgfdeclarelayer{background} +\pgfsetlayers{background,main} + +\tikzset{normal border/.style={orange!30!black!10}} + +% Macro to draw the shape behind the text, when it fits +% completely in the page +\def\parchmentframe#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} % Draw the shape behind + \filldraw[normal border,rounded corners=2em,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Macro to draw the shape, when the text will continue in next page +\def\parchmentframetop#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} + \filldraw[normal border,rounded corners,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Macro to draw the shape, when the text continues from previous page +\def\parchmentframebottom#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} + \filldraw[normal border,rounded corners,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Macro to draw the shape, when both the text continues from previous page +% and it will continue in next page +\def\parchmentframemiddle#1{ +\tikz{ + \node[inner sep=2em] (A) {#1}; % Draw the text of the node + \begin{pgfonlayer}{background} + \filldraw[normal border,rounded corners,color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (A.south east) -- (A.south west) -- + (A.north west) -- (A.north east) -- cycle; + \end{pgfonlayer}}} + +% Define the environment which puts the frame. In this case, +% the environment also accepts an argument with an optional +% title (which defaults to ``Example'', which is typeset in a +% box overlaid on the top border +\newenvironment{parchment}[1][Example]{% + \def\FrameCommand{\parchmentframe}% + \def\FirstFrameCommand{\parchmentframetop}% + \def\LastFrameCommand{\parchmentframebottom}% + \def\MidFrameCommand{\parchmentframemiddle}% + \vskip\baselineskip + \MakeFramed {\FrameRestore} + \noindent\tikz\node[rounded corners=2ex, inner sep=2ex, draw=blue!25!yellow, fill=white, dashed, anchor=west, overlay] at (0em, 2em) {\sffamily#1};\par}% +{\endMakeFramed} + +\setlength\parindent{0pt} + +% Main document, example of usage +\pagestyle{empty} + +\usepackage{enumitem} +\begin{document} +\fontsize{9.5}{11}\selectfont +\setlist[itemize]{leftmargin=0em} + +% We need layers to draw the block diagram +\pgfdeclarelayer{background} +\pgfdeclarelayer{foreground} +\pgfsetlayers{background,main,foreground} + +% Define a few styles and constants +\tikzstyle{left-title}=[draw, fill=blue!20, minimum width=1.6em, minimum height=5em,anchor=north east] +\tikzstyle{top-title}=[draw, fill=blue!20, minimum width=5em, minimum height=1.6em] +\tikzstyle{main-block} = [left-title, text width=30em, inner sep=8pt, fill=red!15!yellow!40, minimum height=5em, rounded corners] +\tikzstyle{immediate} = [left-title, text width=17em, inner sep=8pt, fill=red!15!yellow!40, minimum height=5em, rounded corners] +\tikzstyle{explain} = [left-title, text width=17em, inner sep=8pt, fill=black!5, minimum height=5em, rounded corners,anchor=south west] + +\begin{comment} +\begin{parchment}[commentarii de bello gallico] + Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur. Hi omnes lingua, institutis, legibus inter se differunt. Gallos ab Aquitanis Garumna flumen, a Belgis Matrona et Sequana dividit. Horum omnium fortissimi sunt Belgae, propterea quod a cultu atque humanitate provinciae longissime absunt, minimeque ad eos mercatores saepe commeant atque ea quae ad effeminandos animos pertinent important, proximique sunt Germanis, qui trans Rhenum incolunt, quibuscum continenter bellum gerunt. Qua de causa Helvetii quoque reliquos Gallos virtute praecedunt, quod fere cotidianis proeliis cum Germanis contendunt, cum aut suis finibus eos prohibent aut ipsi in eorum finibus bellum gerunt. Eorum una pars, quam Gallos obtinere dictum est, initium capit a flumine Rhodano, continetur Garumna flumine, Oceano, finibus Belgarum, attingit etiam ab Sequanis et Helvetiis flumen Rhenum, vergit ad septentriones. Belgae ab extremis Galliae finibus oriuntur, pertinent ad inferiorem partem fluminis Rheni, spectant in septentrionem et orientem solem. Aquitania a Garumna flumine ad Pyrenaeos montes et eam partem Oceani quae est ad Hispaniam pertinet; spectat inter occasum solis et septentriones. +\end{parchment} +\end{comment} +\section*{GenServer - a cheat sheet} +\begin{itemize} + \setlength\itemsep{0em} +\item[---] last version: \verb|https://elixir-lang.org/downloads/cheatsheets/gen-server.pdf| +\item[---] reference: \verb|https://hexdocs.pm/elixir/GenServer.html| +\end{itemize} +\begin{comment} + copyright 2019 by Mario Frasca et al. + + based on a work by Benjamin Tan Wei Hao, + rewritten as a tex/tikz document by Mario Frasca, + with José Valim validating the content. +\end{comment} + +\begin{parchment}[initialization: .start $\rightarrow$ \bf\texttt{init/1}] + \begin{tikzpicture} + \node (client) [main-block] {% + \vspace*{-\baselineskip} +\begin{verbatim} +def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, match_this, opts) +end +\end{verbatim} + }; + \path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + + \path (client.north east)+(3em,0) node (immediate) [immediate,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:ok, pid} +:ignore +{:error, term} +\end{verbatim} +}; + \path (immediate.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + + \path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def init(match_this) do + # process input and compute result + result +end +\end{verbatim} +}; + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:ok, state} +{:ok, state, then_what} + +{:stop, reason} +:ignore +\end{verbatim} +}; +\path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result = }}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\path (result.south east)+(3em,0) node (reason) [immediate,anchor=south west,fill=green!30!yellow!18!white] { +One of \verb|:normal|, \verb|:shutdown|, \verb|{:shutdown, _}|, or any other value. See the footnote for a link to the complete reference. +}; +\path (reason.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}reason = }}; +\path (reason.north west)+(0,2pt) node [top-title,anchor=south west,fill=green!25] {applies globally}; + + \draw[->,draw=blue!50!black,dashed,thick,anchor=north west] (-7.3em,-3.3em) .. controls (-7em,-7em) and (-22.45em,-1em) .. (-22.45em,-5.5em) -- (-22.45em,-6.9em); + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[termination: .stop $\rightarrow$ \bf\texttt{terminate/2}] +\begin{tikzpicture} + \node (client) [main-block] {% + \vspace*{-\baselineskip} +\begin{verbatim} +def stop(pid, reason \\ :normal, + timeout \\ :infinity) do + GenServer.stop(pid, reason, timeout) +end +\end{verbatim} +}; +\path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + +\path (client.north east)+(3em,0) node (immediate) [immediate,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +:ok +\end{verbatim} +}; +\path (immediate.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + +\path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def terminate(reason, state) do + # perform cleanup + # result will not be used +end +\end{verbatim} +}; + +\path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + +\path (callback.south east)+(3em,0) node (then-what) [explain] { + \verb|terminate/2| is also called when \verb|:stop| is returned and in case of errors, when \verb|Process.flag(:trap_exit)| is true. +}; + \path (then-what.north west)+(-2pt,0) node [left-title,fill=white] {\rotatebox{90}{:stop}}; + + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[asynchronous operation: .cast $\rightarrow$ \bf\texttt{handle\_cast/2}] +\begin{tikzpicture} + \node (client) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def your_api_async_op(pid, args) do + GenServer.cast(pid, match_this) +end +\end{verbatim} +}; + \path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + +\path (client.north east)+(3em,0) node (result) [immediate,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +:ok +\end{verbatim} +}; + \path (result.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + + \path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_cast(match_this, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \draw[->,draw=blue!50!black,dashed,thick] (-15.1em,-3.3em) .. controls (-15.1em,-5.4em) and (-18.2em,-3em) .. (-18.2em,-5.8em) -- (-18.2em,-6.8em); + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, state} +\end{verbatim} +}; + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + + \path (result.south east)+(3em,0) node (then-what) [immediate,anchor=south west,fill=green!30!yellow!18!white] { + \vspace*{-\baselineskip} +\begin{verbatim} +timeout_milliseconds +:hibernate +{:continue, match_this} +\end{verbatim} +}; +\path (then-what.north west)+(0,2pt) node [top-title,anchor=south west,fill=green!25] {applies globally}; + \path (then-what.south west)+(-2pt,0) node [left-title,anchor=south east,fill=green!25] {\rotatebox{90}{\^{}then\_what =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[synchronous operation: .call $\rightarrow$ \bf\texttt{handle\_call/3}] +\begin{tikzpicture} + \node (client) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def your_api_sync_op(pid, args) do + GenServer.call(pid, match_this) +end +\end{verbatim} +}; + \path (client.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{client}}; + + \path (client.north east)+(3em,0) node (immediate) [immediate,anchor=north west] {waits for callback, receives \verb|reply| if result matches \verb|{:reply, reply, ...}| or \verb|{:stop, _, reply, _}|.}; + \path (immediate.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{returns}}; + + \path (client.south west)+(0,-1em) node (callback) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_call(match_this, from, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \draw[->,draw=blue!50!black,dashed,thick] (-15.1em,-3.3em) .. controls (-15.1em,-5.4em) and (-18.2em,-3em) .. (-18.2em,-5.8em) -- (-18.2em,-6.8em); + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:reply, reply, state} +{:reply, reply, state, then_what} +\end{verbatim} + +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, reply, state} +\end{verbatim} +}; + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + +\path (result.south east)+(3em,0) node (reply) [immediate,anchor=south west] {user defined}; + \path (reply.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}reply =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-72pt) -- ($ (result.north east)+(0,-72pt) $); + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} + +\end{parchment} + +\begin{parchment}[handling messages: $\rightarrow$ \bf\texttt{handle\_info/2}] +\begin{tikzpicture} + \node (callback) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_info(match_this, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, state} +\end{verbatim} +}; + + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} +\end{parchment} + +\begin{parchment}[\bf\texttt{\^{}then\_what = \{:continue, match\_this\}} $\rightarrow$ \bf\texttt{handle\_continue/2}] +\begin{tikzpicture} + \node (callback) [main-block] { + \vspace*{-\baselineskip} +\begin{verbatim} +def handle_continue(match_this, state) do + # process input and compute result + result +end +\end{verbatim} +}; + \path (callback.north west)+(-2pt,0) node [left-title] {\rotatebox{90}{callback}}; + + \path (callback.south west)+(0,-1em) node (result) [main-block,anchor=north west] { + \vspace*{-\baselineskip} +\begin{verbatim} +{:noreply, state} +{:noreply, state, then_what} + +{:stop, reason, state} +\end{verbatim} +}; + + \path (result.north west)+(-2pt,0) node [left-title,fill=green!25] {\rotatebox{90}{\^{}result =}}; + + \draw[color=blue!10!yellow!5,draw=blue!25!yellow,dashed] + (result.north west)+(0,-38pt) -- ($ (result.north east)+(0,-38pt) $); + +\end{tikzpicture} +\end{parchment} +\begin{parchment}[footnotes] + \vspace*{-\baselineskip} + \begin{itemize} + \setlength\itemsep{0em} + \item[---] More on exit reasons: \verb|https://hexdocs.pm/elixir/Supervisor.html#module-exit-reasons-and-restarts| + \item[---] use \verb|@impl true| before each definition to guarantee it matches the equivalent \verb|GenServer| callback. + \item[---] callbacks not listed here are: \verb|code_change/3| and \verb|format_status/2|. + \item[---] source: \verb|https://github.com/elixir-lang/elixir-lang.github.com| + \item[---] copyright: by its authors, listed in the source --- license: CC:BY-SA% + \end{itemize} +\end{parchment} + +\end{document} diff --git a/downloads/logos/elixir-drop-only.png b/downloads/logos/elixir-drop-only.png new file mode 100644 index 000000000..97008feb9 Binary files /dev/null and b/downloads/logos/elixir-drop-only.png differ diff --git a/downloads/logos/elixir-horizontal.png b/downloads/logos/elixir-horizontal.png new file mode 100644 index 000000000..def595473 Binary files /dev/null and b/downloads/logos/elixir-horizontal.png differ diff --git a/downloads/logos/elixir-vertical.png b/downloads/logos/elixir-vertical.png new file mode 100644 index 000000000..52391a543 Binary files /dev/null and b/downloads/logos/elixir-vertical.png differ diff --git a/elixir.csv b/elixir.csv index f9713418f..48961d2c7 100644 --- a/elixir.csv +++ b/elixir.csv @@ -1,7 +1,25 @@ -version,url_precompiled,release_type,windows_installer_compat -1.0.1,https://github.com/elixir-lang/elixir/releases/download/v1.0.1/Precompiled.zip,release,1 -1.0.0,https://github.com/elixir-lang/elixir/releases/download/v1.0.0/Precompiled.zip,release,1 -1.0.0-rc2,https://github.com/elixir-lang/elixir/releases/download/v1.0.0-rc2/Precompiled.zip,release,1 -1.0.0-rc1,https://github.com/elixir-lang/elixir/releases/download/v1.0.0-rc1/Precompiled.zip,release,1 -0.15.1,https://github.com/elixir-lang/elixir/releases/download/v0.15.1/Precompiled.zip,release,1 -0.15.0,https://github.com/elixir-lang/elixir/releases/download/v0.15.0/Precompiled.zip,release,1 +version,url_precompiled,release_type,windows_installer_compat,sha1,sha512,min_otp_release +1.14.5,https://github.com/elixir-lang/elixir/releases/download/v1.14.5/elixir-otp-25.zip,release,1,6fc78c40bd13afbd8531dff6a08846d095c1d4ea,f3b35d9fa61da7e93c9979cb8a08f64a9ce7074aeda66fae994f2a4ea2e4f82e,25.0 +1.14.4,https://github.com/elixir-lang/elixir/releases/download/v1.14.4/elixir-otp-25.zip,release,1,2c52e5a2357030ecb31bbf3dff5835122b1658ab,a5b7aadfd896e691a6494f9079fcd6f1209adcbd93f2d40e5770d80edc0f33e6,25.0 +1.14.2,https://github.com/elixir-lang/elixir/releases/download/v1.14.2/elixir-otp-25.zip,release,1,4bdbb762f102e6318388389a4b44e3a132534ae3,2ab159e875a8d407dde10ee279446d469022acf8128e4f390556ab44e8918a06,25.0 +1.14.1,https://github.com/elixir-lang/elixir/releases/download/v1.14.1/elixir-otp-25.zip,release,1,199e252f71ee82cd2a0026e496f230e9ccd8c1eb,40a880e817bc188469e0adf73f41acb82b6a5b9e5da5cbcb55d89d98dcafe5e5,25.0 +1.14.0,https://github.com/elixir-lang/elixir/releases/download/v1.14.0/elixir-otp-25.zip,release,1,c7ff5116fed9ff7b34f07d3187a8c7c590a79626,55d5708605aa0c81c06acc284e1d40a2853942988f3a6c399759d8ad065e46d0,25.0 +1.13.4,https://github.com/elixir-lang/elixir/releases/download/v1.13.4/Precompiled.zip,release,1,325fbdde4f0a5701bb8b9d455175b85ff41470d7,e64c714e80cd9657b8897d725f6d78f251d443082f6af5070caec863c18068c97af6bdda156c3b3390e0a2b84f77c2ad3378a42913f64bb583fb5251fa49e619,22.0 +1.13.3,https://github.com/elixir-lang/elixir/releases/download/v1.13.3/Precompiled.zip,release,1,7a2d0ff13beadcba3f566d692d960dcd785df5c8,93132c03a16479cfd48c509e2c5ee145b9062d77d528ac2eaeae460f4349f138286f14d34a1ee884e6c76081fe1bf52d27788b944ef06feaa40c07bec41a0a27,22.0 +1.13.2,https://github.com/elixir-lang/elixir/releases/download/v1.13.2/Precompiled.zip,release,1,53bf917f18fc210dcf252e346453fd8c04f2ac6b,74cc0b3d7ddb0156d6695b3b08319d0c01fcfac053407b0bf8b456013d21d1b37ffd5cba389557edfb6af329fbed07c8cd1061a6698b60e1b16a70c898720ec2,22.0 +1.13.1,https://github.com/elixir-lang/elixir/releases/download/v1.13.1/Precompiled.zip,release,1,40762ffbef86cbbfd0c79a94e057fb987dc882ff,bb02ead0d4ccf499ff0473fbbb17fd12de4ba476b463c0452138bc1ef9004547166e75ffe7c6f96b9497adc9fecbda4ad57bfe0f17f1ba95d8339a0e98c29b03,22.0 +1.13.0,https://github.com/elixir-lang/elixir/releases/download/v1.13.0/Precompiled.zip,release,1,0f196f4de406882b5a3bcee7ac9e9b0c61ed1459,58ffe87d6eb89435d8605aee04556aa1a1ba25cf2fa7688d0a5da162d7d7c57b47b2f726b365a7aeb18832bf08de3db5d3ec0ed45e13da276438679f29e5e3ac,22.0 +1.12.3,https://github.com/elixir-lang/elixir/releases/download/v1.12.3/Precompiled.zip,release,1,db3c8a5470dee211168707c432a1f8002960e966,db092caa32b55195eeb24a17e0ab98bb2fea38d2f638bc42fee45a6dfcd3ba0782618d27e281c545651f93914481866b9d34b6d284c7f763d197e87847fdaef4,22.0 +1.12.2,https://github.com/elixir-lang/elixir/releases/download/v1.12.2/Precompiled.zip,release,1,051c2728bfd679d1d6f74d11f25ca679491cd6b4,38eb2281032b0cb096ef5e61f048c5374d6fb9bf4078ab8f9526a42e16e7c661732a632b55d6072328eedf87a47e6eeb3f0e3f90bba1086239c71350f90c75e5,22.0 +1.12.1,https://github.com/elixir-lang/elixir/releases/download/v1.12.1/Precompiled.zip,release,1,4a1269b95c719e0295cd6b57a1c447e1b91563ef,66e877486606f232f36cbe76cdd63bc4ee5e75e27d6292bb602ab2c88b4dd7da6684e6ff9cdb2020e7f916ee2d58016e2d1e2cbd5e579f297ab8eae660cb5dcb,22.0 +1.12.0,https://github.com/elixir-lang/elixir/releases/download/v1.12.0/Precompiled.zip,release,1,b607b2d0e78e1ae6f9d112ba3e1b617a1ee7580b,57e970640f7e273a62fbde60545ed85c18d667e194d0d3483b941ea3164f12664cc0c186968c1f341253f1907157ae88953b5caa413204f93bba08710fdc0b50,22.0 +1.11.4,https://github.com/elixir-lang/elixir/releases/download/v1.11.4/Precompiled.zip,release,1,2b421e7f4b8d62af5c9238f5cf127938d3523f23,4d8ead533a7bd35b41669be0d4548b612d5cc17723da67cfdf996ab36522fd0163215915a970675c6ebcba4dbfc7a46e644cb144b16087bc9417b385955a1e79,21.0 +1.11.3,https://github.com/elixir-lang/elixir/releases/download/v1.11.3/Precompiled.zip,release,1,430735e03d8dfcba3324010d397ab7871b30d87d,16cdd30e41737ba9d659e506b4aa79465ede2d93d27a720d6a3649b8100da9f7dee154a913438724651a83aea77ddef33ac1489bb7ae283b39fbfe20dcd77db4,21.0 +1.11.2,https://github.com/elixir-lang/elixir/releases/download/v1.11.2/Precompiled.zip,release,1,1c54c01b5449d631e673e4aaf205522eaeb8dbdc,7b6ece70d2db6cf4b02c3e72f0ae2edc68e13a2651876525eab08e3c3c3f1f88d294e3f412c3fb736bc84ba4c2897930101b0ac715e322f1554f29c69e2d1e77,21.0 +1.11.1,https://github.com/elixir-lang/elixir/releases/download/v1.11.1/Precompiled.zip,release,1,f03301dd10072f3643a4c1e5f012a49bfc493608,c70e7d3e338ab2449301c7864a2dd96e8d474f3fee533a66706c67cd733db74c2d529b9887961db9f07530eeed4cce917c480e5e8c4201060c7fa494e8b376da,21.0 +1.11.0,https://github.com/elixir-lang/elixir/releases/download/v1.11.0/Precompiled.zip,release,1,4654fed3bbd3e4ebc0c15cfaec003e84080eabd3,6293f2a0ca87872789a9b3909e782cde77409e8b3f1699f42c95ef55360bdba27c69fe6aaa1c843ef004cc4ded9d92e4cc0053f845c0208ec38f1d11945cae07,21.0 +1.10.4,https://github.com/elixir-lang/elixir/releases/download/v1.10.4/Precompiled.zip,release,1,5b400c829c4f239ac89a7eb97aac642b961f38fd,9727ae96d187d8b64e471ff0bb5694fcd1009cdcfd8b91a6b78b7542bb71fca59869d8440bb66a2523a6fec025f1d23394e7578674b942274c52b44e19ba2d43,21.0 +1.10.3,https://github.com/elixir-lang/elixir/releases/download/v1.10.3/Precompiled.zip,release,1,382fa22e6f184c0cc87fcfbfa0053c349d85f6f1,fc6d06ad4cc596b2b6e4f01712f718200c69f3b9c49c7d3b787f9a67b36482658490cf01109b0b0842fc9d88a27f64a9aba817231498d99fa01fa99688263d55,21.0 +1.10.2,https://github.com/elixir-lang/elixir/releases/download/v1.10.2/Precompiled.zip,release,1,830264881231e711d8bade711ed2819a2d10b918,532f43f08a29b5cbfca5a544c337b4a6d336d372ba0a5b3393681426d57ecaa85960e8eb3548aea26f213cc36914c6b66b83707cd719e27dc34c40efadb9f0b9,21.0 +1.10.1,https://github.com/elixir-lang/elixir/releases/download/v1.10.1/Precompiled.zip,release,1,606ffaff6c7c3d278d7daa1fee653aa8183f225f,e8809aff909ca6e2271493690a6a095959e551f6d04e2d384ead0d6cfde23a3707492b53563b9a78080509b44728a63645dc37be108cd6ea3f9501f5d616fe1d,21.0 +1.10.0,https://github.com/elixir-lang/elixir/releases/download/v1.10.0/Precompiled.zip,release,1,0c3f2206131f6d50f9b4e3264dcb8757c52294e4,1e9286391281cd53e5cc5452cdf9ee586c50648800701a1ab80ef7a5a4ef4052f75149235ba348e560d8d5247a0f476c27f481f53e05cbe29244d0b1a25d6586,21.0 diff --git a/erlang.csv b/erlang.csv index 85556905e..7f0e37d55 100644 --- a/erlang.csv +++ b/erlang.csv @@ -1,4 +1,5 @@ version_otp,version_erts,url_win32,url_win64 -17.3,6.2,http://www.erlang.org/download/otp_win32_17.3.exe,http://www.erlang.org/download/otp_win64_17.3.exe -17.1,6.1,http://www.erlang.org/download/otp_win32_17.1.exe,http://www.erlang.org/download/otp_win64_17.1.exe -17,6.0,http://www.erlang.org/download/otp_win32_17.0.exe,http://www.erlang.org/download/otp_win64_17.0.exe +25.3,13.3,https://github.com/erlang/otp/releases/download/OTP-25.3.1/otp_win32_25.3.1.exe,https://github.com/erlang/otp/releases/download/OTP-25.3.1/otp_win64_25.3.1.exe +24.3,12.3,https://github.com/erlang/otp/releases/download/OTP-24.3.4.2/otp_win32_24.3.4.2.exe,https://github.com/erlang/otp/releases/download/OTP-24.3.4.2/otp_win64_24.3.4.2.exe +23.3,11.2,https://github.com/erlang/otp/releases/download/OTP-23.3.4.8/otp_win32_23.3.4.8.exe,https://github.com/erlang/otp/releases/download/OTP-23.3.4.8/otp_win64_23.3.4.8.exe +22.3,10.7,http://www.erlang.org/download/otp_win32_22.3.exe,http://www.erlang.org/download/otp_win64_22.3.exe diff --git a/getting-started/alias-require-and-import.markdown b/getting-started/alias-require-and-import.markdown new file mode 100644 index 000000000..eb771d715 --- /dev/null +++ b/getting-started/alias-require-and-import.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: alias-require-and-import +--- diff --git a/getting-started/basic-operators.markdown b/getting-started/basic-operators.markdown new file mode 100644 index 000000000..459bdf53c --- /dev/null +++ b/getting-started/basic-operators.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: lists-and-tuples +--- diff --git a/getting-started/basic-types.markdown b/getting-started/basic-types.markdown new file mode 100644 index 000000000..2ab92484a --- /dev/null +++ b/getting-started/basic-types.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: basic-types +--- diff --git a/getting-started/binaries-strings-and-char-lists.markdown b/getting-started/binaries-strings-and-char-lists.markdown new file mode 100644 index 000000000..1836f0b2e --- /dev/null +++ b/getting-started/binaries-strings-and-char-lists.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: binaries-strings-and-charlists +--- diff --git a/getting-started/case-cond-and-if.markdown b/getting-started/case-cond-and-if.markdown new file mode 100644 index 000000000..98dd9f544 --- /dev/null +++ b/getting-started/case-cond-and-if.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: case-cond-and-if +--- diff --git a/getting-started/comprehensions.markdown b/getting-started/comprehensions.markdown new file mode 100644 index 000000000..a1ddb4dd7 --- /dev/null +++ b/getting-started/comprehensions.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: comprehensions +--- diff --git a/getting-started/debugging.markdown b/getting-started/debugging.markdown new file mode 100644 index 000000000..c63f3e447 --- /dev/null +++ b/getting-started/debugging.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: debugging +--- diff --git a/getting-started/enumerables-and-streams.markdown b/getting-started/enumerables-and-streams.markdown new file mode 100644 index 000000000..b6abedae7 --- /dev/null +++ b/getting-started/enumerables-and-streams.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: enumerable-and-streams +--- diff --git a/getting-started/erlang-libraries.markdown b/getting-started/erlang-libraries.markdown new file mode 100644 index 000000000..8af923cc7 --- /dev/null +++ b/getting-started/erlang-libraries.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: erlang-libraries +--- diff --git a/getting-started/index.html b/getting-started/index.html new file mode 100644 index 000000000..c50365d75 --- /dev/null +++ b/getting-started/index.html @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: introduction +--- diff --git a/getting-started/introduction.markdown b/getting-started/introduction.markdown new file mode 100644 index 000000000..c50365d75 --- /dev/null +++ b/getting-started/introduction.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: introduction +--- diff --git a/getting-started/io-and-the-file-system.markdown b/getting-started/io-and-the-file-system.markdown new file mode 100644 index 000000000..17a6e7bef --- /dev/null +++ b/getting-started/io-and-the-file-system.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: io-and-the-file-system +--- diff --git a/getting-started/keywords-and-maps.markdown b/getting-started/keywords-and-maps.markdown new file mode 100644 index 000000000..a7430b486 --- /dev/null +++ b/getting-started/keywords-and-maps.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: keywords-and-maps +--- diff --git a/getting-started/meta/domain-specific-languages.markdown b/getting-started/meta/domain-specific-languages.markdown new file mode 100644 index 000000000..96c2c5f2b --- /dev/null +++ b/getting-started/meta/domain-specific-languages.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: domain-specific-languages +--- diff --git a/getting-started/meta/index.html b/getting-started/meta/index.html new file mode 100644 index 000000000..70c16019b --- /dev/null +++ b/getting-started/meta/index.html @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: quote-and-unquote +--- diff --git a/getting-started/meta/macros.markdown b/getting-started/meta/macros.markdown new file mode 100644 index 000000000..349aee294 --- /dev/null +++ b/getting-started/meta/macros.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: macros +--- diff --git a/getting-started/meta/quote-and-unquote.markdown b/getting-started/meta/quote-and-unquote.markdown new file mode 100644 index 000000000..70c16019b --- /dev/null +++ b/getting-started/meta/quote-and-unquote.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: quote-and-unquote +--- diff --git a/getting-started/mix-otp/agent.markdown b/getting-started/mix-otp/agent.markdown new file mode 100644 index 000000000..419cc43f7 --- /dev/null +++ b/getting-started/mix-otp/agent.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: agents +--- diff --git a/getting-started/mix-otp/config-and-releases.markdown b/getting-started/mix-otp/config-and-releases.markdown new file mode 100644 index 000000000..6a141bef9 --- /dev/null +++ b/getting-started/mix-otp/config-and-releases.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: config-and-releases +--- diff --git a/getting-started/mix-otp/dependencies-and-umbrella-apps.markdown b/getting-started/mix-otp/dependencies-and-umbrella-apps.markdown new file mode 100644 index 000000000..946955ce2 --- /dev/null +++ b/getting-started/mix-otp/dependencies-and-umbrella-apps.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: dependencies-and-umbrella-projects +--- diff --git a/getting-started/mix-otp/dependencies-and-umbrella-projects.markdown b/getting-started/mix-otp/dependencies-and-umbrella-projects.markdown new file mode 100644 index 000000000..946955ce2 --- /dev/null +++ b/getting-started/mix-otp/dependencies-and-umbrella-projects.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: dependencies-and-umbrella-projects +--- diff --git a/getting-started/mix-otp/distributed-tasks.markdown b/getting-started/mix-otp/distributed-tasks.markdown new file mode 100644 index 000000000..a75f95d9a --- /dev/null +++ b/getting-started/mix-otp/distributed-tasks.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: distributed-tasks +--- diff --git a/getting-started/mix-otp/docs-tests-and-with.markdown b/getting-started/mix-otp/docs-tests-and-with.markdown new file mode 100644 index 000000000..dbeb73254 --- /dev/null +++ b/getting-started/mix-otp/docs-tests-and-with.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: docs-tests-and-with +--- diff --git a/getting-started/mix-otp/dynamic-supervisor.markdown b/getting-started/mix-otp/dynamic-supervisor.markdown new file mode 100644 index 000000000..9dfcd0300 --- /dev/null +++ b/getting-started/mix-otp/dynamic-supervisor.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: dynamic-supervisor +--- diff --git a/getting-started/mix-otp/ets.markdown b/getting-started/mix-otp/ets.markdown new file mode 100644 index 000000000..22e0a0978 --- /dev/null +++ b/getting-started/mix-otp/ets.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: erlang-term-storage +--- diff --git a/getting-started/mix-otp/genserver.markdown b/getting-started/mix-otp/genserver.markdown new file mode 100644 index 000000000..0c548b6bb --- /dev/null +++ b/getting-started/mix-otp/genserver.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: genservers +--- diff --git a/getting-started/mix-otp/index.html b/getting-started/mix-otp/index.html new file mode 100644 index 000000000..6406e2fed --- /dev/null +++ b/getting-started/mix-otp/index.html @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: introduction-to-mix +--- diff --git a/getting-started/mix-otp/introduction-to-mix.markdown b/getting-started/mix-otp/introduction-to-mix.markdown new file mode 100644 index 000000000..6406e2fed --- /dev/null +++ b/getting-started/mix-otp/introduction-to-mix.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: introduction-to-mix +--- diff --git a/getting-started/mix-otp/supervisor-and-application.markdown b/getting-started/mix-otp/supervisor-and-application.markdown new file mode 100644 index 000000000..2d073b41b --- /dev/null +++ b/getting-started/mix-otp/supervisor-and-application.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: supervisor-and-application +--- diff --git a/getting-started/mix-otp/task-and-gen-tcp.markdown b/getting-started/mix-otp/task-and-gen-tcp.markdown new file mode 100644 index 000000000..a278081ed --- /dev/null +++ b/getting-started/mix-otp/task-and-gen-tcp.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: task-and-gen-tcp +--- diff --git a/getting-started/module-attributes.markdown b/getting-started/module-attributes.markdown new file mode 100644 index 000000000..026b62136 --- /dev/null +++ b/getting-started/module-attributes.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: module-attributes +--- diff --git a/getting-started/modules-and-functions.markdown b/getting-started/modules-and-functions.markdown new file mode 100644 index 000000000..263315f14 --- /dev/null +++ b/getting-started/modules-and-functions.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: modules-and-functions +--- diff --git a/getting-started/modules.markdown b/getting-started/modules.markdown new file mode 100644 index 000000000..263315f14 --- /dev/null +++ b/getting-started/modules.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: modules-and-functions +--- diff --git a/getting-started/optional-syntax.markdown b/getting-started/optional-syntax.markdown new file mode 100644 index 000000000..9231aebf5 --- /dev/null +++ b/getting-started/optional-syntax.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: optional-syntax +--- diff --git a/getting-started/pattern-matching.markdown b/getting-started/pattern-matching.markdown new file mode 100644 index 000000000..bfa53ced1 --- /dev/null +++ b/getting-started/pattern-matching.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: pattern-matching +--- diff --git a/getting-started/processes.markdown b/getting-started/processes.markdown new file mode 100644 index 000000000..1435024b8 --- /dev/null +++ b/getting-started/processes.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: processes +--- diff --git a/getting-started/protocols.markdown b/getting-started/protocols.markdown new file mode 100644 index 000000000..a44082f4e --- /dev/null +++ b/getting-started/protocols.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: protocols +--- diff --git a/getting-started/recursion.markdown b/getting-started/recursion.markdown new file mode 100644 index 000000000..ed7c1c2c9 --- /dev/null +++ b/getting-started/recursion.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: recursion +--- diff --git a/getting-started/sigils.markdown b/getting-started/sigils.markdown new file mode 100644 index 000000000..0f929e472 --- /dev/null +++ b/getting-started/sigils.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: sigils +--- diff --git a/getting-started/structs.markdown b/getting-started/structs.markdown new file mode 100644 index 000000000..2a92104e6 --- /dev/null +++ b/getting-started/structs.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: structs +--- diff --git a/getting-started/try-catch-and-rescue.markdown b/getting-started/try-catch-and-rescue.markdown new file mode 100644 index 000000000..158dc94ce --- /dev/null +++ b/getting-started/try-catch-and-rescue.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: try-catch-and-rescue +--- diff --git a/getting-started/typespecs-and-behaviours.markdown b/getting-started/typespecs-and-behaviours.markdown new file mode 100644 index 000000000..af6ce2592 --- /dev/null +++ b/getting-started/typespecs-and-behaviours.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: typespecs +--- diff --git a/getting-started/where-to-go-next.markdown b/getting-started/where-to-go-next.markdown new file mode 100644 index 000000000..c50365d75 --- /dev/null +++ b/getting-started/where-to-go-next.markdown @@ -0,0 +1,5 @@ +--- +layout: redirect +sitemap: false +redirect_to: introduction +--- diff --git a/getting_started/1.markdown b/getting_started/1.markdown deleted file mode 100644 index b9754753b..000000000 --- a/getting_started/1.markdown +++ /dev/null @@ -1,59 +0,0 @@ ---- -layout: getting_started -title: 1 Introduction -guide: 1 ---- - -# {{ page.title }} - -
      - -Welcome! - -In this tutorial we are going to get you started with Elixir. This chapter will focus on ensuring Elixir is installed and that you can sucessfully run Elixir's Interactive Shell, called IEx. - -Our requirements are: - - * Erlang - Version 17.0 onwards - * Elixir - Version 1.0.0 onwards - -Let's get started! - -> If you find any errors in the tutorial or on the website, [please report a bug or send a pull request to our issue tracker](https://github.com/elixir-lang/elixir-lang.github.com). If you suspect it is a language bug, [please let us know in the language issue tracker](https://github.com/elixir-lang/elixir/issues). - -## 1.1 Installation - -If you still haven't installed Elixir, run to our [installation page](/install.html). Once you are done, you can run `elixir -v` to get the current Elixir version. - -## 1.2 Interactive mode - -When you install Elixir, you will have three new executables: `iex`, `elixir` and `elixirc`. If you compiled Elixir from source or are using a packaged version, you can find these inside the `bin` directory. - -For now, let's start by running `iex` (or `iex.bat` if you are on Windows) which stands for Interactive Elixir. In interactive mode, we can type any Elixir expression and get its result. Let's warm up with some basic expressions: - -```text -Interactive Elixir - press Ctrl+C to exit (type h() ENTER for help) - -iex> 40 + 2 -42 -iex> "hello" <> " world" -"hello world" -``` - -It seems we are ready to go! We will use the interactive shell quite a lot in the next chapters to get a bit more familiar with the language constructs and basic types, starting in the next chapter. - -## 1.3 Running scripts - -After getting familiar with the basics of the language you may want to try writing simple programs. This can be accomplished by putting Elixir code into a file and executing it with `elixir`: - -```text -$ cat simple.exs -IO.puts "Hello world -from Elixir" - -$ elixir simple.exs -Hello world -from Elixir -``` - -Later on we will learn how to compile Elixir code (in [Chapter 8](/getting_started/8.html)) and how to use the Mix build tool (in the [Mix & OTP guide](/getting_started/mix_otp/1.html)). For now, let's move on to [Chapter 2](/getting_started/2.html). diff --git a/getting_started/10.markdown b/getting_started/10.markdown deleted file mode 100644 index a0385407f..000000000 --- a/getting_started/10.markdown +++ /dev/null @@ -1,115 +0,0 @@ ---- -layout: getting_started -title: 10 Enumerables and Streams -guide: 10 ---- - -# {{ page.title }} - -
      -

      - -## 10.1 Enumerables - -Elixir provides the concept of enumerables and [the `Enum` module](/docs/stable/elixir/Enum.html) to work with them. We have already learned two enumerables: lists and maps. - -```iex -iex> Enum.map([1, 2, 3], fn x -> x * 2 end) -[2, 4, 6] -iex> Enum.map(%{1 => 2, 3 => 4}, fn {k, v} -> k * v end) -[2, 12] -``` - -The `Enum` module provides a huge range of functions to transform, sort, group, filter and retrieve items from enumerables. It is one of the modules developers use frequently in their Elixir code. - -Elixir also provides ranges: - -```iex -iex> Enum.map(1..3, fn x -> x * 2 end) -[2, 4, 6] -iex> Enum.reduce(1..3, 0, &+/2) -6 -``` - -Since the Enum module was designed to work across different data types, its API is limited to functions that are useful across many data types. For specific operations, you may need to reach to modules specific to the data types. For example, if you want to insert an element at a given position in a list, you should use the `List.insert_at/3` function from [the `List` module](/docs/stable/elixir/List.html), as it would make little sense to insert a value into, for example, a range. - -We say the functions in the `Enum` module are polymorphic because they can work with diverse data types. In particular, the functions in the `Enum` module can work with any data type that implements [the `Enumerable` protocol](/docs/stable/elixir/Enumerable.html). We are going to discuss Protocols in a later chapter, for now we are going to move on to a specific kind of enumerable called streams. - -## 10.2 Eager vs Lazy - -All the functions in the `Enum` module are eager. Many functions expect an enumerable and return a list back: - -```iex -iex> odd? = &(rem(&1, 2) != 0) -#Function<6.80484245/1 in :erl_eval.expr/5> -iex> Enum.filter(1..3, odd?) -[1, 3] -``` - -This means that when performing multiple operations with `Enum`, each operation is going to generate an intermediate list until we reach the result: - -```iex -iex> 1..100_000 |> Enum.map(&(&1 * 3)) |> Enum.filter(odd?) |> Enum.sum -7500000000 -``` - -The example above has a pipeline of operations. We start with a range and then multiply each element in the range by 3. This first operation will now create and return a list with `100_000` items. Then we keep all odd elements from the list, generating a new list, now with `50_000` items, and then we sum all entries. - -As an alternative, Elixir provides [the `Stream` module](/docs/stable/elixir/Stream.html) which supports lazy operations: - -```iex -iex> 1..100_000 |> Stream.map(&(&1 * 3)) |> Stream.filter(odd?) |> Enum.sum -7500000000 -``` - -Instead of generating intermediate lists, streams create a series of computations that are invoked only when we pass it to the `Enum` module. Streams are useful when working with large, *possibly infinite*, collections. - -## 10.3 Streams - -Streams are lazy, composable enumerables. - -They are lazy because, as shown in the example above, `1..100_000 |> Stream.map(&(&1 * 3))` returns a data type, an actual stream, that represents the `map` computation over the range `1..100_000`: - -```iex -iex> 1..100_000 |> Stream.map(&(&1 * 3)) -#Stream<[enum: 1..100000, funs: [#Function<34.16982430/1 in Stream.map/2>]]> -``` - -Furthermore, they are composable because we can pipe many stream operations: - -```iex -iex> 1..100_000 |> Stream.map(&(&1 * 3)) |> Stream.filter(odd?) -#Stream<[enum: 1..100000, funs: [...]]> -``` - -Many functions in the `Stream` module accept any enumerable as argument and return a stream as result. It also provides functions for creating streams, possibly infinite. For example, `Stream.cycle/1` can be used to create a stream that cycles a given enumerable infinitely. Be careful to not call a function like `Enum.map/2` on such streams, as they would cycle forever: - -```iex -iex> stream = Stream.cycle([1, 2, 3]) -#Function<15.16982430/2 in Stream.cycle/1> -iex> Enum.take(stream, 10) -[1, 2, 3, 1, 2, 3, 1, 2, 3, 1] -``` - -On the other hand, `Stream.unfold/2` can be used to generate values from a given initial value: - -```iex -iex> stream = Stream.unfold("hełło", &String.next_codepoint/1) -#Function<39.75994740/2 in Stream.unfold/2> -iex> Enum.take(stream, 3) -["h", "e", "ł"] -``` - -Another interesting function is `Stream.resource/3` which can be used to wrap around resources, guaranteeing they are opened right before enumeration and closed afterwards, even in case of failures. For example, we can use it to stream a file: - -```iex -iex> stream = File.stream!("path/to/file") -#Function<18.16982430/2 in Stream.resource/3> -iex> Enum.take(stream, 10) -``` - -The example above will fetch the first 10 lines of the file you have selected. This means streams can be very useful for handling large files or even slow resources like network resources. - -The amount of functions and functionality in [`Enum`](/docs/stable/elixir/Enum.html) and [`Stream`](/docs/stable/elixir/Stream.html) modules can be daunting at first but you will get familiar with them case by case. In particular, focus on the `Enum` module first and only move to `Stream` for the particular scenarios where laziness is required to either deal with slow resources or large, possibly infinite, collections. - -Next we'll look at a feature central to Elixir, Processes, which allows us to write concurrent, parallel and distributed programs in an easy and understandable way. diff --git a/getting_started/11.markdown b/getting_started/11.markdown deleted file mode 100644 index 62d9ac990..000000000 --- a/getting_started/11.markdown +++ /dev/null @@ -1,217 +0,0 @@ ---- -layout: getting_started -title: 11 Processes -guide: 11 ---- - -# {{ page.title }} - -
      - -In Elixir, all code runs inside processes. Processes are isolated from each other, run concurrent to one another and communicate via message passing. Processes are not only the basis for concurrency in Elixir, but they also provide the means for building distributed and fault-tolerant programs. - -Elixir's processes should not be confused with operating system processes. Processes in Elixir are extremely lightweight in terms of memory and CPU (unlike threads in many other programming languages). Because of this, it is not uncommon to have dozens of thousands of processes running simultaneously. - -In this chapter, we will learn about the basic constructs for spawning new processes, as well as sending and receiving messages between different processes. - -## 11.1 spawn - -The basic mechanism for spawning new processes is with the auto-imported `spawn/1` function: - -```iex -iex> spawn fn -> 1 + 2 end -#PID<0.43.0> -``` - -`spawn/1` takes a function which it will execute in another process. - -Notice `spawn/1` returns a PID (process identifier). At this point, the process you spawned is very likely dead. The spawned process will execute the given function and exit after the function is done: - -```iex -iex> pid = spawn fn -> 1 + 2 end -#PID<0.44.0> -iex> Process.alive?(pid) -false -``` - -> Note: you will likely get different process identifiers than the ones we are getting in this guide. - -We can retrieve the PID of the current process by calling `self/0`: - -```iex -iex> self() -#PID<0.41.0> -iex> Process.alive?(self()) -true -``` - -Processes get much more interesting when we are able to send and receive messages. - -## 11.2 send and receive - -We can send messages to a process with `send/2` and receive them with `receive/1`: - -```iex -iex> send self(), {:hello, "world"} -{:hello, "world"} -iex> receive do -...> {:hello, msg} -> msg -...> {:world, msg} -> "won't match" -...> end -"world" -``` - -When a message is sent to a process, the message is stored in the process mailbox. The `receive/1` block goes through the current process mailbox searching for a message that matches any of the given patterns. `receive/1` supports many clauses, like `case/2`, as well as guards in the clauses. - -If there is no message in the mailbox matching any of the patterns, the current process will wait until a matching message arrives. A timeout can also be specified: - -```iex -iex> receive do -...> {:hello, msg} -> msg -...> after -...> 1_000 -> "nothing after 1s" -...> end -"nothing after 1s" -``` - -A timeout of 0 can be given when you already expect the message to be in the mailbox. - -Let's put all together and send messages between processes: - -```iex -iex> parent = self() -#PID<0.41.0> -iex> spawn fn -> send(parent, {:hello, self()}) end -#PID<0.48.0> -iex> receive do -...> {:hello, pid} -> "Got hello from #{inspect pid}" -...> end -"Got hello from #PID<0.48.0>" -``` - -While in the shell, you may find the helper `flush/0` quite useful. It flushes and prints all the messages in the mailbox. - -```iex -iex> send self(), :hello -:hello -iex> flush() -:hello -:ok -``` - -## 11.3 Links - -The most common form of spawning in Elixir is actually via `spawn_link/1`. Before we show an example with `spawn_link/1`, let's try to see what happens when a process fails: - -```iex -iex> spawn fn -> raise "oops" end - -[error] Error in process <0.58.0> with exit value: ... - -#PID<0.58.0> -``` - -It merely logged an error but the spawning process is still running. That's because processes are isolated. If we want the failure in one process to propagate to another one, we should link them. This can be done with `spawn_link/1`: - -```iex -iex> spawn_link fn -> raise "oops" end - -** (EXIT from #PID<0.41.0>) an exception was raised: - ** (RuntimeError) oops - :erlang.apply/2 -``` - -When a failure happens in the shell, the shell automatically traps the failure and shows it nicely formatted. In order to understand what would really happen in our code, let's use `spawn_link/1` inside a file and run it: - -```iex -# spawn.exs -spawn_link fn -> raise "oops" end - -receive do - :hello -> "let's wait until the process fails" -end -``` - -This time the process failed and brought the parent process down as they are linked. Linking can also be done manually by calling `Process.link/1`. We recommend you to take a look at [the `Process` module](/docs/stable/elixir/Process.html) for other functionality provided by processes. - -Process and links play an important role when building fault-tolerant systems. In Elixir applications, we often link our processes to supervisors which will detect when a process dies and start a new process in its place. This is only possible because processes are isolated and don't share anything by default. And if processes are isolated, there is no way a failure in a process will crash or corrupt the state of another. - -While other languages would require us to catch/handle exceptions, in Elixir we are actually fine with letting processes fail because we expect supervisors to properly restart our systems. "Failing fast" is a common philosophy when writing Elixir software! - -Before moving to the next chapter, let's see one of the most common use cases for creating processes in Elixir. - -## 11.4 State - -We haven't talked about state so far in this guide. If you are building an application that requires state, for example, to keep your application configuration, or you need to parse a file and keep it in memory, where would you store it? - -Processes are the most common answer to this question. We can write processes that loop infinitely, maintain state, and send and receive messages. As an example, let's write a module that starts new processes that work as a key-value store in a file named `kv.exs`: - -```elixir -defmodule KV do - def start do - {:ok, spawn_link(fn -> loop(%{}) end)} - end - - defp loop(map) do - receive do - {:get, key, caller} -> - send caller, Map.get(map, key) - loop(map) - {:put, key, value} -> - loop(Map.put(map, key, value)) - end - end -end -``` - -Note that the `start` function basically spawns a new process that runs the `loop/1` function, starting with an empty map. The `loop/1` function then waits for messages and performs the appropriate action for each message. In case of a `:get` message, it sends a message back to the caller and calls `loop/1` again, to wait for a new message. While the `:put` message actually invokes `loop/1` with a new version of the map, with the given `key` and `value` stored. - -Let's give it a try by running `iex kv.exs`: - -```iex -iex> {:ok, pid} = KV.start -#PID<0.62.0> -iex> send pid, {:get, :hello, self()} -{:get, :hello, #PID<0.41.0>} -iex> flush -nil -``` - -At first, the process map has no keys, so sending a `:get` message and then flushing the current process inbox returns `nil`. Let's send a `:put` message and try it again: - -```iex -iex> send pid, {:put, :hello, :world} -#PID<0.62.0> -iex> send pid, {:get, :hello, self()} -{:get, :hello, #PID<0.41.0>} -iex> flush -:world -``` - -Notice how the process is keeping a state and we can get and update this state by sending the process messages. In fact, any process that knows the `pid` above will be able to send it messages and manipulate the state. - -It is also possible to register the `pid`, giving it a name, and allowing everyone that knows the name to send it messages: - -```iex -iex> Process.register(pid, :kv) -true -iex> send :kv, {:get, :hello, self()} -{:get, :hello, #PID<0.41.0>} -iex> flush -:world -``` - -Using processes around state and name registering are very common patterns in Elixir applications. However, most of the time, we won't implement those patterns manually as above, but by using one of the many of the abstractions that ships with Elixir. For example, Elixir provides [agents](/docs/stable/elixir/Agent.html) which are simple abstractions around state: - -```iex -iex> {:ok, pid} = Agent.start_link(fn -> %{} end) -{:ok, #PID<0.72.0>} -iex> Agent.update(pid, fn map -> Map.put(map, :hello, :world) end) -:ok -iex> Agent.get(pid, fn map -> Map.get(map, :hello) end) -:world -``` - -A `:name` option could also be given to `Agent.start/2` and it would be automatically registered. Besides agents, Elixir provides an API for building generic servers (called GenServer), generic event managers and event handlers (called GenEvent), tasks and more, all powered by processes underneath. Those, along with supervision trees, will be explored with more detail in the Mix and OTP guide which will build a complete Elixir application from start to finish. - -For now, let's move on and explore the world of I/O in Elixir. diff --git a/getting_started/12.markdown b/getting_started/12.markdown deleted file mode 100644 index c811f1d50..000000000 --- a/getting_started/12.markdown +++ /dev/null @@ -1,175 +0,0 @@ ---- -layout: getting_started -title: 12 IO -guide: 12 ---- - -# {{ page.title }} - -
      - -This chapter is a quick introduction to input/output mechanisms in Elixir and related modules, like [`IO`](/docs/stable/elixir/IO.html), [`File`](/docs/stable/elixir/File.html) and [`Path`](/docs/stable/elixir/Path.html). - -We had originally sketched this chapter to come much earlier in the getting started guide. However, we noticed the IO system provides a great opportunity to shed some light on some philosophies and curiosities of Elixir and the VM. - -## 12.1 The IO module - -The `IO` module in Elixir is the main mechanism for reading and writing to the standard io (`:stdio`), standard error (`:stderr`), files and other IO devices. Usage of the module is pretty straight-forward: - -```iex -iex> IO.puts "hello world" -"hello world" -:ok -iex> IO.gets "yes or no? " -yes or no? yes -"yes\n" -``` - -By default, the functions in the IO module use the standard input and output. We can pass the `:stderr` as argument to write to the standard error device: - -```iex -iex> IO.puts :stderr, "hello world" -"hello world" -:ok -``` - -## 12.2 The File module - -The [`File`](/docs/stable/elixir/File.html) module contains functions that allows us to open files as IO devices. By default, files are opened in binary mode, which requires developers to use the specific `IO.binread/2` and `IO.binwrite/2` functions from the `IO` module: - -```iex -iex> {:ok, file} = File.open "hello", [:write] -{:ok, #PID<0.47.0>} -iex> IO.binwrite file, "world" -:ok -iex> File.close file -:ok -iex> File.read "hello" -{:ok, "world"} -``` - -A file can also be opened with `:utf8` encoding which allows the remaining functions in the `IO` module to be used: - -```iex -iex> {:ok, file} = File.open "another", [:write, :utf8] -{:ok, #PID<0.48.0>} -``` - -Besides functions for opening, reading and writing files, the `File` module has many functions that work on the file system. Those functions are named after their UNIX equivalents. For example, `File.rm/1` can be used to remove files, `File.mkdir/1` to create directories, `File.mkdir_p/1` creates directories guaranteeing their parents exists and there is even `File.cp_r/2` and `File.rm_rf/2` which copy and remove files and directories recursively. - -You will also notice that functions in the `File` module have two variants, one with `!` (bang) in its name and others without. For example, when we read the "hello" file above, we have used the one without `!`. Let's try some new examples: - -```iex -iex> File.read "hello" -{:ok, "world"} -iex> File.read! "hello" -"world" -iex> File.read "unknown" -{:error, :enoent} -iex> File.read! "unknown" -** (File.Error) could not read file unknown: no such file or directory -``` - -Notice that when the file does not exist, the version with `!` raises an error. That said, the version without `!` is preferred when you want to handle different outcomes with pattern matching. However, if you expect the file to be there, the bang variation is more useful as it raises a meaningful error message. That said, never write: - -```elixir -{:ok, body} = File.read(file) -``` - -Instead write: - -```elixir -case File.read(file) do - {:ok, body} -> # handle ok - {:error, r} -> # handle error -end -``` - -or - -```elixir -File.read!(file) -``` - -## 12.3 The Path module - -The majority of the functions in the File module expects paths as arguments. Most commonly, those paths will be binaries and they can be manipulated with the [`Path`](/docs/stable/elixir/Path.html) module: - -```iex -iex> Path.join("foo", "bar") -"foo/bar" -iex> Path.expand("~/hello") -"/Users/jose/hello" -``` - -With this we have covered the main modules for doing IO and interacting with the file system. Next we will discuss some curiosities and advanced topics regarding IO. Those sections are not necessary to write Elixir code, so feel free to skip them, but they do provide an overview of how the IO system is implemented in the VM and other curiosities. - -## 12.4 Processes and group leaders - -You may have noticed that `File.open/2` returned a tuple containing a PID: - -```iex -iex> {:ok, file} = File.open "hello", [:write] -{:ok, #PID<0.47.0>} -``` - -That's because the IO module actually works with processes. When you say `IO.write(pid, binary)`, the IO module will send a message to the process with the desired operation. Let's see what happens if we use our own process: - -```iex -iex> pid = spawn fn -> -...> receive do: (msg -> IO.inspect msg) -...> end -#PID<0.57.0> -iex> IO.write(pid, "hello") -{:io_request, #PID<0.41.0>, #PID<0.57.0>, {:put_chars, :unicode, "hello"}} -** (ErlangError) erlang error: :terminated -``` - -After `IO.write/2`, we can see the request sent by the IO module printed, which then fails since the IO module expected some kind of result that we did not supply. - -The [`StringIO`](/docs/stable/elixir/StringIO.html) module provides an implementation of the IO device messages on top of a string: - -```iex -iex> {:ok, pid} = StringIO.open("hello") -{:ok, #PID<0.43.0>} -iex> IO.read(pid, 2) -"he" -``` - -By modelling IO devices with processes, the Erlang VM allows different nodes in the same network to exchange file processes to read/write files in between nodes. Of all IO devices, there is one that is special to each process, called group leader. - -When you write to `:stdio`, you are actually sending a message to the group leader, which writes to STDIO file descriptor: - -```iex -iex> IO.puts :stdio, "hello" -hello -:ok -iex> IO.puts Process.group_leader, "hello" -hello -:ok -``` - -The group leader can be configured per process and is used in different situations. For example, when executing code in a remote terminal, it guarantees messages in a remote node are redirected and printed in the terminal that triggered the request. - -## 12.5 `iodata` and `chardata` - -In all examples above, we have used binaries/strings when writing to files. In the chapter "Binaries, strings and char lists", we mentioned how strings are simply bytes while char lists are lists with code points. - -The functions in `IO` and `File` also allow lists to be given as arguments. Not only that, they also allow a mixed list of lists, integers and binaries to be given: - -```iex -iex> IO.puts 'hello world' -hello world -:ok -iex> IO.puts ['hello', ?\s, "world"] -hello world -:ok -``` - -However, this requires some attention. A list may represent either a bunch of bytes or a bunch of characters and which one to use depends on the encoding of the IO device. If the file is opened without encoding, the file is expected to be in raw mode, and the functions in the `IO` module starting with `bin*` must be used. Those functions expect an `iodata` as argument, i.e. it expects a list of integers representing bytes and binaries to be given. - -On the other hand, `:stdio` and files opened with `:utf8` encoding work with the remaining functions in the `IO` module and those expect a `char_data` as argument, i.e. they expect a list of characters or strings to be given. - -Although this is a subtle difference, you only need to worry about those details if you intend to pass lists to those functions. Binaries are already represented by the underlying bytes and as such their representation is always raw. - -This finishes our tour of IO devices and IO related functionality. We have learned about four Elixir modules, [`IO`](/docs/stable/elixir/IO.html), [`File`](/docs/stable/elixir/File.html), [`Path`](/docs/stable/elixir/Path.html) and [`StringIO`](/docs/stable/elixir/StringIO.html), as well as how the VM uses processes for the underlying IO mechanisms and how to use (char and io) data for IO operations. diff --git a/getting_started/13.markdown b/getting_started/13.markdown deleted file mode 100644 index fa8c84745..000000000 --- a/getting_started/13.markdown +++ /dev/null @@ -1,179 +0,0 @@ ---- -layout: getting_started -title: 13 alias, require and import -guide: 13 ---- - -# {{ page.title }} - -
      - -In order to facilitate software reuse, Elixir provides three directives. As we are going to see below, they are called directives because they have **lexical scope**. - -## 13.1 alias - -`alias` allows you to set up aliases for any given module name. Imagine our `Math` module uses a special list implementation for doing math specific operations: - -```elixir -defmodule Math do - alias Math.List, as: List -end -``` - -From now on, any reference to `List` will automatically expand to `Math.List`. In case one wants to access the original `List`, it can be done by accessing the module via `Elixir`: - -```elixir -List.flatten #=> uses Math.List.flatten -Elixir.List.flatten #=> uses List.flatten -Elixir.Math.List.flatten #=> uses Math.List.flatten -``` - -> Note: All modules defined in Elixir are defined inside a main Elixir namespace. However, for convenience, you can omit "Elixir." when referencing them. - -Aliases are frequently used to define shortcuts. In fact, calling `alias` without an `:as` option sets the alias automatically to the last part of the module name, for example: - -```elixir -alias Math.List -``` - -Is the same as: - -```elixir -alias Math.List, as: List -``` - -Note that `alias` is **lexically scoped**, which allows you to set aliases inside specific functions: - -```elixir -defmodule Math do - def plus(a, b) do - alias Math.List - # ... - end - - def minus(a, b) do - # ... - end -end -``` - -In the example above, since we are invoking `alias` inside the function `plus/2`, the alias will just be valid inside the function `plus/2`. `minus/2` won't be affected at all. - -## 13.2 require - -Elixir provides macros as a mechanism for meta-programming (writing code that generates code). - -Macros are chunks of code that are executed and expanded at compilation time. This means, in order to use a macro, we need to guarantee its module and implementation are available during compilation. This is done with the `require` directive: - -```iex -iex> Integer.is_odd(3) -** (CompileError) iex:1: you must require Integer before invoking the macro Integer.is_odd/1 -iex> require Integer -nil -iex> Integer.is_odd(3) -true -``` - -In Elixir, `Integer.is_odd/1` is defined as a macro so it can be used as guards. This means that, in order to invoke `Integer.is_odd/1`, we need to first require the `Integer` module. - -In general a module does not need to be required before usage, except if we want to use the macros available in that module. An attempt to call a macro that was not loaded will raise an error. Note that like the `alias` directive, `require` is also lexically scoped. We will talk more about macros in a later chapter. - -## 13.3 import - -We use `import` whenever we want to easily access functions or macros from other modules without using the qualified name. For instance, if we want to use the `duplicate` function from `List` several times, we can simply import it: - -```iex -iex> import List, only: [duplicate: 2] -nil -iex> duplicate :ok, 3 -[:ok, :ok, :ok] -``` - -In this case, we are importing only the function `duplicate` (with arity 2) from `List`. Although `:only` is optional, its usage is recommended. `:except` could also be given as an option. - -`import` also supports `:macros` and `:functions` to be given to `:only`. For example, to import all macros, one could write: - -```elixir -import Integer, only: :macros -``` - -Or to import all functions, you could write: - -```elixir -import Integer, only: :functions -``` - -Note that `import` is also **lexically scoped**, this means we can import specific macros inside specific functions: - -```elixir -defmodule Math do - def some_function do - import List, only: [duplicate: 2] - # call duplicate - end -end -``` - -In the example above, the imported `List.duplicate/2` is only visible within that specific function. `duplicate/2` won't be available in any other function in that module (or any other module for that matter). - -Note that importing a module automatically requires it. - -## 13.4 Aliases - -At this point, you may be wondering, what exactly an Elixir alias is and how it is represented? - -An alias in Elixir is a capitalized identifier (like `String`, `Keyword`, etc) which is converted to an atom during compilation. For instance, the `String` alias translates by default to the atom `:"Elixir.String"`: - -```iex -iex> is_atom(String) -true -iex> to_string(String) -"Elixir.String" -iex> :"Elixir.String" -String -``` - -By using the `alias/2` directive, we are simply changing what an alias translates to. - -Aliases work as described because in the Erlang VM (and consequently Elixir), modules are represented by atoms. For example, that's the mechanism we use to call Erlang modules: - -```iex -iex> :lists.flatten([1,[2],3]) -[1, 2, 3] -``` - -This is also the mechanism that allows us to dynamically call a given function in a module: - -```iex -iex> mod = :lists -:lists -iex> mod.flatten([1,[2],3]) -[1,2,3] -``` - -In other words, we are simply calling the function `flatten` on the atom `:lists`. - -## 13.5 Nesting - -Now that we have talked about aliases, we can talk about nesting and how it works in Elixir. Consider the following example: - -```elixir -defmodule Foo do - defmodule Bar do - end -end -``` - -The example above will define two modules `Foo` and `Foo.Bar`. The second can be accessed as `Bar` inside `Foo` as long as they are in the same lexical scope. If later the developer decides to move `Bar` to another file, it will need to be referenced by its full name (`Foo.Bar`) or an alias needs to be set using the `alias` directive discussed above. - -In other words, the code above is exactly the same as: - -```elixir -defmodule Elixir.Foo do - defmodule Elixir.Foo.Bar do - end - alias Elixir.Foo.Bar, as: Bar -end -``` - -As we will see in later chapters, aliases also play a crucial role in macros, to guarantee they are hygienic. With this we are almost finishing our tour about Elixir modules, the last topic to cover is module attributes. diff --git a/getting_started/14.markdown b/getting_started/14.markdown deleted file mode 100644 index a9eede613..000000000 --- a/getting_started/14.markdown +++ /dev/null @@ -1,167 +0,0 @@ ---- -layout: getting_started -title: 14 Module attributes -guide: 14 ---- - -# {{ page.title }} - -
      - -Module attributes in Elixir serve three purposes: - -1. They serve to annotate the module, often with information to be used by the user or the VM. -2. They work as constants. -3. They work as a temporary module storage to be used during compilation. - -Let's check each case, one by one. - -## 14.1 As annotations - -Elixir brings the concept of module attributes from Erlang. For example: - -```elixir -defmodule MyServer do - @vsn 2 -end -``` - -In the example above, we are explicitly setting the version attribute for that module. `@vsn` is used by the code reloading mechanism in the Erlang VM to check if a module has been updated or not. If no version is specified, the version is set to the MD5 checksum of the module functions. - -Elixir has a handful of reserved attributes. Here are just a few of them, the most commonly used ones: - -* `@moduledoc` - provides documentation for the current module. -* `@doc` - provides documentation for the function or macro that follows the attribute. -* `@behaviour` - (notice the British spelling) used for specifying an OTP or user-defined behaviour. -* `@before_compile` - provides a hook that will be invoked before the module is compiled. This makes it possible to inject functions inside the module exactly before compilation. - -`@moduledoc` and `@doc` are by far the most used attributes, and we expect you to use them a lot. Elixir treats documentation as first-class and provides many functions to access documentation. - -Let's go back to the `Math` module defined in the previous chapters, add some documentation and save it to the `math.ex` file: - -```elixir -defmodule Math do - @moduledoc """ - Provides math-related functions. - - ## Examples - - iex> Math.sum(1, 2) - 3 - - """ - - @doc """ - Calculates the sum of two numbers. - """ - def sum(a, b), do: a + b -end -``` - -Elixir promotes the use of markdown with heredocs to write readable documentation. Heredocs are multiline strings, they start and end with triple quotes, keeping the formatting of the inner text. We can access the documentation of any compiled module directly from IEx: - - $ elixirc math.ex - $ iex - iex> h Math # Access the docs for the module Math - ... - iex> h Math.sum # Access the docs for the sum function - ... - -We also provide a tool called [ExDoc](https://github.com/elixir-lang/ex_doc) which is used to generate HTML pages from the documentation. - -You can take a look at the docs for [Module](/docs/stable/elixir/Module.html) for a complete list of supported attributes. Elixir also uses attributes to define [typespecs](/docs/stable/elixir/Kernel.Typespec.html), via: - -* `@spec` - provides a specification for a function. -* `@callback` - provides a specification for the behaviour callback. -* `@type` - defines a type to be used in `@spec`. -* `@typep` - defines a private type to be used in `@spec`. -* `@opaque` - defines an opaque type to be used in `@spec`. - -This section covers built-in attributes. However, attributes can also be used by developers or extended by libraries to support custom behaviour. - -## 14.2 As constants - -Elixir developers will often use module attributes to be used as constants: - -```elixir -defmodule MyServer do - @initial_state %{host: "147.0.0.1", port: 3456} - IO.inspect @initial_state -end -``` - -> Note: Unlike Erlang, user defined attributes are not stored in the module by default. The value exists only during compilation time. A developer can configure an attribute to behave closer to Erlang by calling [`Module.register_attribute/3`](/docs/stable/elixir/Module.html#register_attribute/3). - -Trying to access an attribute that was not defined will print a warning: - -```elixir -defmodule MyServer do - @unknown -end -warning: undefined module attribute @unknown, please remove access to @unknown or explicitly set it to nil before access -``` - -Finally, attributes can also be read inside functions: - -```elixir -defmodule MyServer do - @my_data 14 - def first_data, do: @my_data - @my_data 13 - def second_data, do: @my_data -end - -MyServer.first_data #=> 14 -MyServer.second_data #=> 13 -``` - -Notice that reading an attribute inside a function takes a snapshot of its current value. In other words, the value is read at compilation time and not at runtime. As we are going to see, this makes attributes useful to be used as storage during module compilation. - -## 14.3 As temporary storage - -One of the projects in the Elixir organization is [the `Plug` project](https://github.com/elixir-lang/plug), which is meant to be a common foundation for building web libraries and frameworks in Elixir. - -The Plug library also allows developers to define their own plugs which can be run in a web server: - -```elixir -defmodule MyPlug do - use Plug.Builder - - plug :set_header - plug :send_ok - - def set_header(conn, _opts) do - put_resp_header(conn, "x-header", "set") - end - - def send_ok(conn, _opts) do - send(conn, 200, "ok") - end -end - -IO.puts "Running MyPlug with Cowboy on http://localhost:4000" -Plug.Adapters.Cowboy.http MyPlug, [] -``` - -In the example above, we have used the `plug/1` macro to connect functions that will be invoked when there is a web request. Internally, every time you call `plug/1`, the Plug library stores the given argument in a `@plugs` attribute. Just before the module is compiled, Plug runs a callback that defines a method (`call/2`) which handles http requests. This method will run all plugs inside `@plugs` in order. - -In order to understand the underlying code, we'd need macros, so we will revisit this pattern in the meta-programming guide. However the focus here is exactly on how using module attributes as storage allow developers to create DSLs. - -Another example comes from the ExUnit framework which uses module attributes as annotation and storage: - -```elixir -defmodule MyTest do - use ExUnit.Case - - @tag :external - test "contacts external service" do - # ... - end -end -``` - -Tags in ExUnit are used to annotate tests. Tags can be later used to filter tests. For example, you can avoid running external tests on your machine because they are slow and dependent on other services, while they can still be enabled in your build system. - -We hope this section shines some light on how Elixir supports meta-programming and how module attributes play an important role when doing so. - -In the next chapters we'll explore structs and protocols before moving to exception handling and other constructs like sigils and comprehensions. diff --git a/getting_started/15.markdown b/getting_started/15.markdown deleted file mode 100644 index d216652bd..000000000 --- a/getting_started/15.markdown +++ /dev/null @@ -1,110 +0,0 @@ ---- -layout: getting_started -title: 15 Structs -guide: 15 ---- - -# {{ page.title }} - -
      - -In earlier chapters, we learned about maps: - -```iex -iex> map = %{a: 1, b: 2} -%{a: 1, b: 2} -iex> map[:a] -1 -iex> %{map | a: 3} -%{a: 3, b: 2} -``` - -Structs are extensions on top of maps that bring default values, compile-time guarantees and polymorphism into Elixir. - -To define a struct, we just need to call `defstruct/1` inside a module: - -```iex -iex> defmodule User do -...> defstruct name: "john", age: 27 -...> end -{:module, User, - <<70, 79, 82, ...>>, {:__struct__, 0}} -``` - -We can now create "instances" of this struct by using the `%User{}` syntax: - -```iex -iex> %User{} -%User{age: 27, name: "john"} -iex> %User{name: "meg"} -%User{age: 27, name: "meg"} -iex> is_map(%User{}) -true -``` - -Structs give compile-time guarantees that the provided fields exist in the struct: - -```iex -iex> %User{oops: :field} -** (CompileError) iex:3: unknown key :oops for struct User -``` - -When discussing maps, we demonstrated how we can access and update existing fields of a map. The same applies to structs: - -```iex -iex> john = %User{} -%User{age: 27, name: "john"} -iex> john.name -"john" -iex> meg = %{john | name: "meg"} -%User{age: 27, name: "meg"} -iex> %{meg | oops: :field} -** (ArgumentError) argument error -``` - -By using the update syntax, the VM is aware no new keys will be added to the map/struct, allowing the maps to share their structure in memory. In the example above, both `john` and `meg` share the same key structure in memory. - -Structs can also be used in pattern matching and they guarantee the structs are of the same type: - -```iex -iex> %User{name: name} = john -%User{age: 27, name: "john"} -iex> name -"john" -iex> %User{} = %{} -** (MatchError) no match of right hand side value: %{} -``` - -Matching works because structs store a field named `__struct__` inside the map: - -```iex -iex> john.__struct__ -User -``` - -Overall, a struct is just a bare map with default fields. Notice we say it is a bare map because none of the protocols implemented for maps are available for structs. For example, you can't enumerate nor access a struct: - -```iex -iex> user = %User{} -%User{age: 27, name: "john"} -iex> user[:name] -** (Protocol.UndefinedError) protocol Access not implemented for %User{age: 27, name: "john"} -``` - -A struct also is not a dictionary and therefore can't be used with the `Dict` module: - -```iex -iex> Dict.get(%User{}, :name) -** (ArgumentError) unsupported dict: %User{name: "john", age: 27} -``` - -Since structs are just maps, they will work with the `Map` module: - -```iex -iex> Map.put(%User{}, :name, "kurt") -%User{age: 27, name: "kurt"} -iex> Map.merge(%User{age: 27}, %User{name: "takashi"}) -%User{age: 27, name: "takashi"} -``` - -We will cover how structs interacts with protocols in the next chapter. diff --git a/getting_started/16.markdown b/getting_started/16.markdown deleted file mode 100644 index d5d6b26f7..000000000 --- a/getting_started/16.markdown +++ /dev/null @@ -1,198 +0,0 @@ ---- -layout: getting_started -title: 16 Protocols -guide: 16 ---- - -# {{ page.title }} - -
      - -Protocols are a mechanism to achieve polymorphism in Elixir. Dispatching on a protocol is available to any data type as long as it implements the protocol. Let's see an example. - -In Elixir, only `false` and `nil` are treated as false. Everything else evaluates to true. Depending on the application, it may be important to specify a `blank?` protocol that returns a boolean for other data types that should be considered blank. For instance, an empty list or an empty binary could be considered blanks. - -We could define this protocol as follows: - -```elixir -defprotocol Blank do - @doc "Returns true if data is considered blank/empty" - def blank?(data) -end -``` - -The protocol expects a function called `blank?` that receives one argument to be implemented. We can implement this protocol for different Elixir data types as follows: - -```elixir -# Integers are never blank -defimpl Blank, for: Integer do - def blank?(_), do: false -end - -# Just empty list is blank -defimpl Blank, for: List do - def blank?([]), do: true - def blank?(_), do: false -end - -# Just empty map is blank -defimpl Blank, for: Map do - # Keep in mind we could not pattern match on %{} because - # it matches on all maps. We can however check if the size - # is zero (and size is a fast operation). - def blank?(map), do: map_size(map) == 0 -end - -# Just the atoms false and nil are blank -defimpl Blank, for: Atom do - def blank?(false), do: true - def blank?(nil), do: true - def blank?(_), do: false -end -``` - -And we would do so for all native data types. The types available are: - -* `Atom` -* `BitString` -* `Float` -* `Function` -* `Integer` -* `List` -* `Map` -* `PID` -* `Port` -* `Reference` -* `Tuple` - -Now with the protocol defined and implementations in hand, we can invoke it: - -```iex -iex> Blank.blank?(0) -false -iex> Blank.blank?([]) -true -iex> Blank.blank?([1, 2, 3]) -false -``` - -Passing a data type that does not implement the protocol raises an error: - -```iex -iex> Blank.blank?("hello") -** (Protocol.UndefinedError) protocol Blank not implemented for "hello" -``` - -## 16.1 Protocols and structs - -The power of Elixir's extensibility comes when protocols and structs are used together. - -In the previous chapter, we have learned that although structs are maps, they do not share protocol implementations with maps. Let's define a `User` struct as in the previous chapter: - -```iex -iex> defmodule User do -...> defstruct name: "john", age: 27 -...> end -{:module, User, - <<70, 79, 82, ...>>, {:__struct__, 0}} -``` - -And then check: - -```iex -iex> Blank.blank?(%{}) -true -iex> Blank.blank?(%User{}) -** (Protocol.UndefinedError) protocol Blank not implemented for %User{age: 27, name: "john"} -``` - -Instead of sharing protocol implementation with maps, structs require their own protocol implementation: - -```elixir -defimpl Blank, for: User do - def blank?(_), do: false -end -``` - -If desired, you could come up with your own semantics for a user being blank. Not only that, you could use structs to build more robust data types, like queues, and implement all relevant protocols, such as `Enumerable` and possibly `Blank`, for this data type. - -In many cases though, developers may want to provide a default implementation for structs, as explicitly implementing the protocol for all structs can be tedious. That's when falling back to Any comes in handy. - -## 16.2 Falling back to Any - -It may be convenient to provide a default implementation for all types. This can be achieved by setting `@fallback_to_any` to `true` in the protocol definition: - -```elixir -defprotocol Blank do - @fallback_to_any true - def blank?(data) -end -``` - -Which can now be implemented as: - -```elixir -defimpl Blank, for: Any do - def blank?(_), do: false -end -``` - -Now all data types (including structs) that we have not implemented the `Blank` protocol for will be considered non-blank. - -## 16.3 Built-in protocols - -Elixir ships with some built-in protocols. In previous chapters, we have discussed the `Enum` module which provides many functions that work with any data structure that implements the `Enumerable` protocol: - -```iex -iex> Enum.map [1, 2, 3], fn(x) -> x * 2 end -[2,4,6] -iex> Enum.reduce 1..3, 0, fn(x, acc) -> x + acc end -6 -``` -Another useful example is the `String.Chars` protocol, which specifies how to convert a data structure with characters to a string. It's exposed via the `to_string` function: - -```iex -iex> to_string :hello -"hello" -``` - -Notice that string interpolation in Elixir calls the `to_string` function: - -```iex -iex> "age: #{25}" -"age: 25" -``` - -The snippet above only works because numbers implement the `String.Chars` protocol. Passing a tuple, for example, will lead to an error: - -```iex -iex> tuple = {1, 2, 3} -{1, 2, 3} -iex> "tuple: #{tuple}" -** (Protocol.UndefinedError) protocol String.Chars not implemented for {1, 2, 3} -``` - -When there is a need to "print" a more complex data structure, one can simply use the `inspect` function, based on the `Inspect` protocol: - -```iex -iex> "tuple: #{inspect tuple}" -"tuple: {1, 2, 3}" -``` - -The `Inspect` protocol is the protocol used to transform any data structure into a readable textual representation. This is what tools like IEx use to print results: - -```iex -iex> {1, 2, 3} -{1,2,3} -iex> %User{} -%User{name: "john", age: 27} -``` - -Keep in mind that, by convention, whenever the inspected value starts with `#`, it is representing a data structure in non-valid Elixir syntax. This means the inspect protocol is not reversible as information may be lost along the way: - -```iex -iex> inspect &(&1+2) -"#Function<6.71889879/1 in :erl_eval.expr/5>" -``` - -There are other protocols in Elixir but this covers the most common ones. In the next chapter we will learn a bit more about error handling and exceptions in Elixir. diff --git a/getting_started/17.markdown b/getting_started/17.markdown deleted file mode 100644 index 111efce98..000000000 --- a/getting_started/17.markdown +++ /dev/null @@ -1,179 +0,0 @@ ---- -layout: getting_started -title: 17 try, catch and rescue -guide: 17 ---- - -# {{ page.title }} - -
      - -Elixir has three error mechanisms: errors, throws and exits. In this chapter we will explore each of them and include remarks about when each should be used. - -## 17.1 Errors - -A sample error can be retrieved by trying to add a number into an atom: - -```iex -iex> :foo + 1 -** (ArithmeticError) bad argument in arithmetic expression - :erlang.+(:foo, 1) -``` - -A runtime error can be raised any time by using the `raise/1` macro: - -```iex -iex> raise "oops" -** (RuntimeError) oops -``` - -Other errors can be raised with `raise/2` passing the error name and a list of keyword arguments: - -```iex -iex> raise ArgumentError, message: "invalid argument foo" -** (ArgumentError) invalid argument foo -``` - -You can also define your own errors by creating a module and use the `defexception/1` macro inside it. The most common case is to define an exception with a message field: - -```iex -iex> defmodule MyError do -iex> defexception message: "default message" -iex> end -iex> raise MyError -** (MyError) default message -iex> raise MyError, message: "custom message" -** (MyError) custom message -``` - -Exceptions can be rescued by using the `try/rescue` construct: - -```iex -iex> try do -...> raise "oops" -...> rescue -...> e in RuntimeError -> e -...> end -%RuntimeError{message: "oops"} -``` - -The example above rescues the runtime error and returns the error itself which is then printed in the `iex` session. In practice Elixir developers rarely use the `try/rescue` construct though. For example, many languages would force you to rescue an error when a file cannot open successfully. Elixir instead provides a `File.read/1` function which returns a tuple containing information if the file was opened with success or not: - -```iex -iex> File.read "hello" -{:error, :enoent} -iex> File.write "hello", "world" -:ok -iex> File.read "hello" -{:ok, "world"} -```` - -There is no `try/rescue` here. In case you want to handle multiple outcomes of opening a file, you can simply use pattern matching with `case`: - -```iex -iex> case File.read "hello" do -...> {:ok, body} -> IO.puts "got ok" -...> {:error, body} -> IO.puts "got error" -...> end -``` - -At the end of the day, it is up to your application to decide if an error while opening a file is exceptional or not. That's why Elixir doesn't impose exceptions on `File.read/1` and many other functions. Instead we leave it up to the developer to choose the best way to proceed. - -For the cases where you do expect a file to exist (and the lack of a file is truly an error) you can simply use `File.read!/1`: - -```iex -iex> File.read! "unknown" -** (File.Error) could not read file unknown: no such file or directory - (elixir) lib/file.ex:305: File.read!/1 -``` - -In other words, we avoid using `try/rescue` because **we don't use errors for control flow**. In Elixir, we take errors literally: they are reserved to unexpected and/or exceptional situations. In case you actually need flow control constructs, throws must be used. That's what we are going to see next. - -## 17.2 Throws - -In Elixir, one can throw a value to be caught later. `throw` and `catch` are reserved for situations where it is not possible to retrieve a value unless by using `throw` and `catch`. - -Those situations are quite uncommon in practice unless when interfacing with a library that does not provide the proper APIs. For example, let's imagine the `Enum` module did not provide any API for finding a value and we need to find the first number that is a multiple of 13: - -```iex -iex> try do -...> Enum.each -50..50, fn(x) -> -...> if rem(x, 13) == 0, do: throw(x) -...> end -...> "Got nothing" -...> catch -...> x -> "Got #{x}" -...> end -"Got -39" -``` - -However, in practice one can simply use `Enum.find/2`: - -```iex -iex> Enum.find -50..50, &(rem(&1, 13) == 0) --39 -``` - -## 17.3 Exits - -Every Elixir code runs inside processes that communicates with each other. When a process dies, it sends an `exit` signal. A process can also die by explicitly sending an exit signal: - -```iex -iex> spawn_link fn -> exit(1) end -#PID<0.56.0> -** (EXIT from #PID<0.56.0>) 1 -``` - -In the example above, the linked process died by sending an `exit` signal with value of 1. The Elixir shell automatically handles those messages and prints them to the terminal. - -`exit` can also be "caught" using `try/catch`: - -```iex -iex> try do -...> exit "I am exiting" -...> catch -...> :exit, _ -> "not really" -...> end -"not really" -``` - -Using `try/catch` is already uncommon and using it to catch exits is even more rare. - -`exit` signals are an important part of the fault tolerant system provided by the Erlang VM. Processes usually run under supervision trees which are themselves processes that just wait for `exit` signals of the supervised processes. Once an exit signal is received, the supervision strategy kicks in and the supervised process is restarted. - -It is exactly this supervision system that makes constructs like `try/catch` and `try/rescue` so uncommon in Elixir. Instead of rescuing a certain error, we'd rather "fail fast" since the supervision tree will guarantee our application will go back to a known initial state after the error. - -## 17.4 After - -Sometimes it is necessary to use `try/after` to guarantee a resource is cleaned up after some particular action. For example, we can open a file and guarantee it is closed with `try/after` block: - -```iex -iex> {:ok, file} = File.open "sample", [:utf8, :write] -iex> try do -...> IO.write file, "olá" -...> raise "oops, something went wrong" -...> after -...> File.close(file) -...> end -** (RuntimeError) oops, something went wrong -``` - -## 17.5 Variables scope - -It is important to bear in mind that variables defined inside `try/catch/rescue/after` blocks do not leak to the outer context. This is because the `try` block may fail and as such the variables may never be bound in the first place. In other words, this code is invalid: - -```iex -iex> try do -...> from_try = true -...> after -...> from_after = true -...> end -iex> from_try -** (RuntimeError) undefined function: from_try/0 -iex> from_after -** (RuntimeError) undefined function: from_after/0 -``` - -This finishes our introduction to `try`, `catch` and `rescue`. You will find they are used less frequently in Elixir than in other languages although they may be handy in some situations where a library or some particular code is not playing "by the rules". - -It is time to talk about some Elixir constructs like comprehensions and sigils. diff --git a/getting_started/18.markdown b/getting_started/18.markdown deleted file mode 100644 index 1758b2c90..000000000 --- a/getting_started/18.markdown +++ /dev/null @@ -1,96 +0,0 @@ ---- -layout: getting_started -title: 18 Comprehensions -guide: 18 ---- - -# {{ page.title }} - -
      - -In Elixir, it is common to loop over Enumerables, often filtering some results, and mapping to another list of values. Comprehensions are syntax sugar for such constructs, grouping those common tasks into the `for` special form. - -For example, we can get all the square values of elements in a list as follows: - -```iex -iex> for n <- [1, 2, 3, 4], do: n * n -[1, 4, 9, 16] -``` - -A comprehension is made of three parts: generators, filters and collectables. - -## 18.1 Generators and filters - -In the expression above, `n <- [1, 2, 3, 4]` is the generator. It is literally generating values to be used in the comprehensions. Any enumerable can be passed in the right-hand side the generator expression: - -```iex -iex> for n <- 1..4, do: n * n -[1, 4, 9, 16] -``` - -Generator expressions also support pattern matching, ignoring all non-matching patterns. Imagine that instead of a range, we have a keyword list where the key is the atom `:good` or `:bad` and we only want to calculate the square of the good values: - -```iex -iex> values = [good: 1, good: 2, bad: 3, good: 4] -iex> for {:good, n} <- values, do: n * n -[1, 4, 16] -``` - -Alternatively, filters can be used to filter some particular elements out. For example, we can get the square of only odd numbers: - -```iex -iex> require Integer -iex> for n <- 1..4, Integer.is_odd(n), do: n * n -[1, 9] -``` - -A filter will keep all values except `nil` or `false`. - -Comprehensions in general provide a much more concise representation than using the equivalent functions from the `Enum` and `Stream` modules. Furthermore, comprehensions also allow multiple generators and filters to be given. Here is an example that receives a list of directories and deletes all files in those directories: - -```elixir -for dir <- dirs, - file <- File.ls!(dir), - path = Path.join(dir, file), - File.regular?(path) do - File.rm!(path) -end -``` - -Keep in mind that variable assignments inside the comprehension, be it in generators, filters or inside the block, are not reflected outside of the comprehension. - -## 18.2 Bitstring generators - -Bitstring generators are also supported and are very useful when you need to organize bitstring streams. The example below receives a list of pixels from a binary with their respective red, green and blue values and convert them into triplets: - -```iex -iex> pixels = <<213, 45, 132, 64, 76, 32, 76, 0, 0, 234, 32, 15>> -iex> for <>, do: {r, g, b} -[{213,45,132},{64,76,32},{76,0,0},{234,32,15}] -``` - -A bitstring generator can be mixed with the "regular" enumerable generators and provide filters as well. - -## 18.3 Into - -In the examples above, the comprehension returned a list as a result. - -However, the result of a comprehension can be inserted into different data structures by passing the `:into` option. For example, we can use bitstring generators with the `:into` option to easily remove all spaces in a string: - -```iex -iex> for <>, c != ?\s, into: "", do: <> -"helloworld" -``` - -Sets, maps and other dictionaries can also be given with the `:into` option. In general, the `:into` accepts any structure as long as it implements the `Collectable` protocol. - -For example, the `IO` module provides streams, that are both `Enumerable` and `Collectable`. You can implement an echo terminal that returns whatever is typed, but in upcase, using comprehensions: - -```iex -iex> stream = IO.stream(:stdio, :line) -iex> for line <- stream, into: stream do -...> String.upcase(line) <> "\n" -...> end -``` - -Now type any string into the terminal and you will see the same value will be printed in upcase. Unfortunately, this example also got your shell stuck in the comprehension, so you will need to hit `Ctrl+C` twice to get out of it. :) diff --git a/getting_started/19.markdown b/getting_started/19.markdown deleted file mode 100644 index c4b6e3f38..000000000 --- a/getting_started/19.markdown +++ /dev/null @@ -1,183 +0,0 @@ ---- -layout: getting_started -title: 19 Sigils -guide: 19 ---- - -# {{ page.title }} - -
      - -We have already learned Elixir provides double-quoted strings and single-quoted char lists. However, this only covers the surface of structures that have textual representation in the language. Atoms are, for example, another structure which are mostly created via the `:atom` representation. - -One of Elixir's goals is extensibility: developers should be able to extend the language to particular domains. Computer science has become such a wide field that it is impossible for a language to tackle many fields as part of its core. Our best bet is to rather make the language extensible, so developers, companies and communities can extend the language to their relevant domains. - -In the chapter, we are going to explore sigils, which are one of the mechanisms provided by the language for working with textual representations. - -## 19.1 Regular expressions - -Sigils start with the tilde (`~`) character which is followed by a letter and then a separator. The most common sigil in Elixir is `~r` for [regular expressions](https://en.wikipedia.org/wiki/Regular_Expressions): - -```iex -# A regular expression that returns true if the text has foo or bar -iex> regex = ~r/foo|bar/ -~r/foo|bar/ -iex> "foo" =~ regex -true -iex> "bat" =~ regex -false -``` - -Elixir provides Perl-compatible regular expressions (regexes), as implemented by the [PCRE](http://www.pcre.org/) library. Regexes also support modifiers. For example, the `i` modifier makes a regular expression case insensitive: - -```iex -iex> "HELLO" =~ ~r/hello/ -false -iex> "HELLO" =~ ~r/hello/i -true -``` - -Check out the [`Regex` module](/docs/stable/elixir/Regex.html) for more information on other modifiers and the supported operations with regular expressions. - -So far, all examples have used `/` to delimit a regular expression. However sigils support 8 different separators: - -``` -~r/hello/ -~r|hello| -~r"hello" -~r'hello' -~r(hello) -~r[hello] -~r{hello} -~r -``` - -The reasoning in supporting different operators is that different separators can be more convenient to different sigils. For example, using parentheses for regular expressions may be a confusing choice as they can get mixed with the parentheses inside the regex. However, parentheses can be handy for other sigils, as we will see in the next section. - -## 19.2 Strings, char lists and words sigils - -Besides regular expressions, Elixir ships with three other sigils. - -The `~s` sigil is used to generate strings, similar to double quotes: - -```iex -iex> ~s(this is a string with "quotes") -"this is a string with \"quotes\"" -``` - -While `~c` is used to generate char lists: - -```iex -iex> ~c(this is a string with "quotes") -'this is a string with "quotes"' -``` - -The `~w` sigil is used to generate a list of words separated by white space: - -```iex -iex> ~w(foo bar bat) -["foo", "bar", "bat"] -``` - -The `~w` sigil also accepts the `c`, `s` and `a` modifiers (for char lists, strings and atoms, respectively) to choose the format of the result: - -```iex -iex> ~w(foo bar bat)a -[:foo, :bar, :bat] -``` - -Besides lowercase sigils, Elixir supports uppercase sigils. While both `~s` and `~S` will return strings, the first one allows escape codes and interpolation while the second does not: - -```elixir -iex> ~s(String with escape codes \x26 interpolation) -"String with escape codes & interpolation" -iex> ~S(String without escape codes and without #{interpolation}) -"String without escape codes and without \#{interpolation}" -``` - -The following escape codes can be used in strings and char lists: - -* `\"` – double quote -* `\'` – single quote -* `\\` – single backslash -* `\a` – bell/alert -* `\b` – backspace -* `\d` - delete -* `\e` - escape -* `\f` - form feed -* `\n` – newline -* `\r` – carriage return -* `\s` – space -* `\t` – tab -* `\v` – vertical tab -* `\DDD`, `\DD`, `\D` - character with octal representation DDD, DD or D (example: `\377`) -* `\xDD` - character with hexadecimal representation DD (example: `\x13`) -* `\x{D...}` - character with hexadecimal representation with one or more hexadecimal digits (example: `\x{abc13}`) - -Sigils also support heredocs, which is when triple double- or single-quotes are used as separators: - -```iex -iex> ~s""" -...> this is -...> a heredoc string -...> """ -``` - -The most common case for heredoc sigils is when writing documentation. For example, if you need to write escape characters in your documentation, it can become error prone as we would need to double-escape some characters: - -```elixir -@doc """ -Converts double-quotes to single-quotes. - -## Examples - - iex> convert("\\\"foo\\\"") - "'foo'" - -""" -def convert(...) -``` - -By using using `~S`, we can avoid this problem altogether: - -```elixir -@doc ~S""" -Converts double-quotes to single-quotes. - -## Examples - - iex> convert("\"foo\"") - "'foo'" - -""" -def convert(...) -``` - -## 19.3 Custom sigils - -As hinted at the beginning of this chapter, sigils in Elixir are extensible. In fact, the sigil `~r/foo/i` is equivalent to calling the `sigil_r` function with two arguments: - -```iex -iex> sigil_r(<<"foo">>, 'i') -~r"foo"i -``` - -That said, we can access the documentation for the `~r` sigil via the `sigil_r` function: - -```iex -iex> h sigil_r -... -``` - -We can also provide our own sigils by simply implementing the proper function. For example, let's implement the `~i(13)` sigil that returns an integer: - -```iex -iex> defmodule MySigils do -...> def sigil_i(string, []), do: String.to_integer(string) -...> end -iex> import MySigils -iex> ~i(13) -13 -``` - -Sigils can also be used to do compile-time work with the help of macros. For example, regular expressions in Elixir are compiled into efficient representation during compilation of the source code, therefore skipping this step at runtime. If you have interest in the subject, we recommend you to learn more about macros and check how those sigils are implemented in the `Kernel` module. diff --git a/getting_started/2.markdown b/getting_started/2.markdown deleted file mode 100644 index 96de3540c..000000000 --- a/getting_started/2.markdown +++ /dev/null @@ -1,366 +0,0 @@ ---- -layout: getting_started -title: 2 Basic types -guide: 2 ---- - -# {{ page.title }} - -
      - -In this chapter we will learn more about Elixir basic types: integers, floats, booleans, atoms, strings, lists and tuples. Some basic types are: - -```iex -iex> 1 # integer -iex> 0x1F # integer -iex> 1.0 # float -iex> true # boolean -iex> :atom # atom / symbol -iex> "elixir" # string -iex> [1, 2, 3] # list -iex> {1, 2, 3} # tuple -``` - -## 2.1 Basic arithmetic - -Open up `iex` and type the following expressions: - -```iex -iex> 1 + 2 -3 -iex> 5 * 5 -25 -iex> 10 / 2 -5.0 -``` - -Notice that `10 / 2` returned a float `5.0` instead of an integer `5`. This is expected. In Elixir, the operator `/` always returns a float. If you want to do integer division or get the division remainder, you can invoke the `div` and `rem` functions: - -```iex -iex> div(10, 2) -5 -iex> div 10, 2 -5 -iex> rem 10, 3 -1 -``` - -Notice that parentheses are not required in order to invoke a function. - -Elixir also supports shortcut notations for entering binary, octal and hexadecimal numbers: - -```iex -iex> 0b1010 -10 -iex> 0o777 -511 -iex> 0x1F -31 -``` - -Float numbers require a dot followed by at least one digit and also support `e` for the exponent number: - -```iex -iex> 1.0 -1.0 -iex> 1.0e-10 -1.0e-10 -``` - -Floats in Elixir are 64 bit double precision. - -You can invoke the `round` function to get the closest integer to a given float, or the `trunc` function to get the integer part of a float. - -```iex -iex> round 3.58 -4 -iex> trunc 3.58 -3 -``` - -## 2.2 Booleans - -Elixir supports `true` and `false` as booleans: - -```iex -iex> true -true -iex> true == false -false -``` - -Elixir provides a bunch of predicate functions to check for a value type. For example, the `is_boolean/1` function can be used to check if a value is a boolean or not: - -> Note: Functions in Elixir are identified by name and by number of arguments (i.e. arity). Therefore, `is_boolean/1` identifies a function named `is_boolean` that takes 1 argument. `is_boolean/2` identifies a different (nonexistent) function with the same name but different arity. - -```iex -iex> is_boolean(true) -true -iex> is_boolean(1) -false -``` - -You can also use `is_integer/1`, `is_float/1` or `is_number/1` to check, respectively, if an argument is an integer, a float or either. - -> Note: At any moment you can type `h` in the shell to print information on how to use the shell. The `h` helper can also be used to access documentation for any function. For example, typing `h is_integer/1` is going to print the documentation for the `is_integer/1` function. It also works with operators and other constructs (try `h ==/2`). - -## 2.3 Atoms - -Atoms are constants where their name is their own value. Some other languages call these symbols: - -```iex -iex> :hello -:hello -iex> :hello == :world -false -``` - -The booleans `true` and `false` are, in fact, atoms: - -```iex -iex> true == :true -true -iex> is_atom(false) -true -``` - -## 2.4 Strings - -Strings in Elixir are inserted between double quotes, and they are encoded in UTF-8: - -```iex -iex> "hellö" -"hellö" -``` - -Elixir also supports string interpolation: - -```iex -iex> "hellö #{:world}" -"hellö world" -``` - -Strings can have line breaks in them or introduce them using escape sequences: - -```iex -iex> "hello -...> world" -"hello\nworld" -iex> "hello\nworld" -"hello\nworld" -``` - -You can print a string using the `IO.puts/1` function from the `IO` module: - -```iex -iex> IO.puts "hello\nworld" -hello -world -:ok -``` - -Notice the `IO.puts/1` function returns the atom `:ok` as result after printing. - -Strings in Elixir are represented internally by binaries which are sequences of bytes: - -```iex -iex> is_binary("hellö") -true -``` - -We can also get the number of bytes in a string: - -```iex -iex> byte_size("hellö") -6 -``` - -Notice the number of bytes in that string is 6, even though it has 5 characters. That's because the character "ö" takes 2 bytes to be represented in UTF-8. We can get the actual length of the string, based on the number of characters, by using the `String.length/1` function: - -```iex -iex> String.length("hellö") -5 -``` - -The [String module](/docs/stable/elixir/String.html) contains a bunch of functions that operate on strings as defined in the Unicode standard: - -```iex -iex> String.upcase("hellö") -"HELLÖ" -``` - -Keep in mind `single-quoted` and `double-quoted` strings are not equivalent in Elixir as they are represented by different types: - -```iex -iex> 'hellö' == "hellö" -false -``` - -We will talk more about Unicode support and the difference between single and double-quoted strings in the "Binaries, strings and char lists" chapter. - -## 2.5 Anonymous functions - -Functions are delimited by the keywords `fn` and `end`: - -```iex -iex> add = fn a, b -> a + b end -#Function<12.71889879/2 in :erl_eval.expr/5> -iex> is_function(add) -true -iex> is_function(add, 2) -true -iex> is_function(add, 1) -false -iex> add.(1, 2) -3 -``` - -Functions are "first class citizens" in Elixir meaning they can be passed as arguments to other functions just as integers and strings can. In the example, we have passed the function in the variable `add` to the `is_function/1` function which correctly returned `true`. We can also check the arity of the function by calling `is_function/2`. - -Note a dot (`.`) between the variable and parenthesis is required to invoke an anonymous function. - -Anonymous functions are closures, and as such they can access variables that are in scope when the function is defined: - -```iex -iex> add_two = fn a -> add.(a, 2) end -#Function<6.71889879/1 in :erl_eval.expr/5> -iex> add_two.(2) -4 -``` - -Keep in mind that a variable assigned inside a function does not affect its surrounding environment: - -```iex -iex> x = 42 -42 -iex> (fn -> x = 0 end).() -0 -iex> x -42 -``` - -## 2.6 (Linked) Lists - -Elixir uses square brackets to specify a list of values. Values can be of any type: - -```iex -iex> [1, 2, true, 3] -[1, 2, true, 3] -iex> length [1, 2, 3] -3 -``` - -Two lists can be concatenated and subtracted using the `++/2` and `--/2` operators: - -```iex -iex> [1, 2, 3] ++ [4, 5, 6] -[1, 2, 3, 4, 5, 6] -iex> [1, true, 2, false, 3, true] -- [true, false] -[1, 2, 3, true] -``` - -Throughout the tutorial, we will talk a lot about the head and tail of a list. The head is the first element of a list and the tail is the remainder of a list. They can be retrieved with the functions `hd/1` and `tl/1`. Let's assign a list to a variable and retrieve its head and tail: - -```iex -iex> list = [1,2,3] -iex> hd(list) -1 -iex> tl(list) -[2, 3] -``` - -Getting the head or the tail of an empty list is an error: - -```iex -iex> hd [] -** (ArgumentError) argument error -``` - -Oops! - -## 2.7 Tuples - -Elixir uses curly brackets to define tuples. Like lists, tuples can hold any value: - -```iex -iex> {:ok, "hello"} -{:ok, "hello"} -iex> tuple_size {:ok, "hello"} -2 -``` - -Tuples store elements contiguously in memory. This means accessing a tuple element per index or getting the tuple size is a fast operation (indexes start from zero): - -```iex -iex> tuple = {:ok, "hello"} -{:ok, "hello"} -iex> elem(tuple, 1) -"hello" -iex> tuple_size(tuple) -2 -``` - -It is also possible to set an element at a particular index in a tuple with `put_elem/3`: - -```iex -iex> tuple = {:ok, "hello"} -{:ok, "hello"} -iex> put_elem(tuple, 1, "world") -{:ok, "world"} -iex> tuple -{:ok, "hello"} -``` - -Notice that `put_elem/3` returned a new tuple. The original tuple stored in the `tuple` variable was not modified because Elixir data types are immutable. By being immutable, Elixir code is easier to reason about as you never need to worry if a particular code is mutating your data structure in place. - -By being immutable, Elixir also helps eliminate common cases where concurrent code has race conditions because two different entities are trying to change a data structure at the same time. - -## 2.8 Lists or tuples? - -What is the difference between lists and tuples? - -Lists are stored in memory as linked lists. This means each element in a list points to the next element, and then to the next element, until it reaches the end of a list. We call each of those pairs in a list a **cons cell**: - -```iex -iex> list = [1|[2|[3|[]]]] -[1, 2, 3] -``` - -This means accessing the length of a list is a linear operation: we need to traverse the whole list in order to figure out its size. Updating a list is fast as long as we are prepending elements: - -```iex -iex> [0] ++ list -[0, 1, 2, 3] -iex> list ++ [4] -[1, 2, 3, 4] -``` - -The first operation is fast because we are simply adding a new cons that points to the remaining of `list`. The second one is slow because we need to rebuild the whole list and add a new element to the end. - -Tuples, on the other hand, are stored contiguously in memory. This means getting the tuple size or accessing an element by index is fast. However, updating or adding elements to tuples is expensive because it requires copying the whole tuple in memory. - -Those performance characteristics dictate the usage of those data structures. One very common use case for tuples is to use them to return extra information from a function. For example, `File.read/1` is a function that can be used to read file contents and it returns tuples: - -```iex -iex> File.read("path/to/existing/file") -{:ok, "... contents ..."} -iex> File.read("path/to/unknown/file") -{:error, :enoent} -``` - -If the path given to `File.read/1` exists, it returns a tuple with the atom `:ok` as the first element and the file contents as the second. Otherwise, it returns a tuple with `:error` and the error description. - -Most of the time, Elixir is going to guide you to do the right thing. For example, there is a `elem/2` function to access a tuple item but there is no built-in equivalent for lists: - -```iex -iex> tuple = {:ok, "hello"} -{:ok, "hello"} -iex> elem(tuple, 1) -"hello" -``` - -When "counting" the number of elements in a data structure, Elixir also abides by a simple rule: the function should be named `size` if the operation is in constant time (i.e. the value is pre-calculated) or `length` if the operation requires explicit counting. - -For example, we have used 4 counting functions so far: `byte_size/1` (for the number of bytes in a string), `tuple_size/1` (for the tuple size), `length/1` (for the list length) and `String.length/1` (for the number of characters in a string). That said, we use `byte_size` to get the number of bytes in a string, which is cheap, but retrieving the number of unicode characters uses `String.length`, since the whole string needs to be iterated. - -Elixir also provides `Port`, `Reference` and `PID` as data types (usually used in process communication), and we will take a quick look at them when talking about processes. For now, let's take a look at some of the basic operators that go with our basic types. diff --git a/getting_started/20.markdown b/getting_started/20.markdown deleted file mode 100644 index 4bb667b09..000000000 --- a/getting_started/20.markdown +++ /dev/null @@ -1,44 +0,0 @@ ---- -layout: getting_started -title: 20 Where to go next -guide: 20 -last: true ---- - -# {{ page.title }} - -
      - -Eager to learn more? Keep on reading! - -## 20.1 Build your first Elixir project - -In order to get your first project started, Elixir ships with a build tool called Mix. You can get your new project started by simply running: - - mix new path/to/new/project - -We have written a guide that covers how to build an Elixir application, with its own supervision tree, configuration, tests and more. The application works as a distributed key-value store where we organize key-value pairs into buckets and distribute those buckets across multiple nodes: - -* [Mix and OTP](/getting_started/mix_otp/1.html) - -## 20.2 Community and other resources - -On the sidebar, you can find the link to some Elixir books and screencasts. There are plenty of Elixir resources out there, like conference talks, open source projects, and other learning materials produced by the community. - -Remember that in case of any difficulties, you can always visit the **#elixir-lang** channel on **irc.freenode.net** or send a message to the [mailing list](http://groups.google.com/group/elixir-lang-talk). You can be sure that there will be someone willing to help. And to keep posted on the latest news and announcements, follow the [blog](http://elixir-lang.org/blog/) and follow language developments on the [elixir-core mailing list](http://groups.google.com/group/elixir-lang-core). - -Don't forget [you can also check the source code of Elixir itself](https://github.com/elixir-lang/elixir), which is mostly written in Elixir (mainly the `lib` directory), or [explore Elixir's documentation](http://elixir-lang.org/docs.html). - -## 20.3 A Byte of Erlang - -As the main page of this site puts it: - -> Elixir is a programming language built on top of the Erlang VM. - -Sooner or later, an Elixir developer will want to interface with existing Erlang libraries. Here's a list of online resources that cover Erlang's fundamentals and its more advanced features: - -* This [Erlang Syntax: A Crash Course](http://elixir-lang.org/crash-course.html) provides a concise intro to Erlang's syntax. Each code snippet is accompanied by equivalent code in Elixir. This is an opportunity for you to not only get some exposure to Erlang's syntax but also review some of the things you have learned in this guide. - -* Erlang's official website has a short [tutorial](http://www.erlang.org/course/concurrent_programming.html) with pictures that briefly describe Erlang's primitives for concurrent programming. - -* [Learn You Some Erlang for Great Good!](http://learnyousomeerlang.com/) is an excellent introduction to Erlang, its design principles, standard library, best practices and much more. If you are serious about Elixir, you'll want to get a solid understanding of Erlang principles. Once you have read through the crash course mentioned above, you'll be able to safely skip the first couple of chapters in the book that mostly deal with the syntax. When you reach [The Hitchhiker's Guide to Concurrency](http://learnyousomeerlang.com/the-hitchhikers-guide-to-concurrency) chapter, that's where the real fun starts. diff --git a/getting_started/3.markdown b/getting_started/3.markdown deleted file mode 100644 index cb17c874b..000000000 --- a/getting_started/3.markdown +++ /dev/null @@ -1,116 +0,0 @@ ---- -layout: getting_started -title: 3 Basic operators -guide: 3 ---- - -# {{ page.title }} - -
      - -In the previous chapter, we saw Elixir provides `+`, `-`, `*`, `/` as arithmetic operators, plus the functions `div/2` and `rem/2` for integer division and remainder. - -Elixir also provides `++` and `--` to manipulate lists: - -```iex -iex> [1,2,3] ++ [4,5,6] -[1,2,3,4,5,6] -iex> [1,2,3] -- [2] -[1,3] -``` - -String concatenation is done with `<>`: - -```iex -iex> "foo" <> "bar" -"foobar" -``` - -Elixir also provides three boolean operators: `or`, `and` and `not`. These operators are strict in the sense that they expect a boolean (`true` or `false`) as their first argument: - -```iex -iex> true and true -true -iex> false or is_atom(:example) -true -``` - -Providing a non-boolean will raise an exception: - -```iex -iex> 1 and true -** (ArgumentError) argument error -``` - -`or` and `and` are short-circuit operators. They only execute the right side if the left side is not enough to determine the result: - -```iex -iex> false and error("This error will never be raised") -false - -iex> true or error("This error will never be raised") -true -``` - -> Note: If you are an Erlang developer, `and` and `or` in Elixir actually map to the `andalso` and `orelse` operators in Erlang. - -Besides these boolean operators, Elixir also provides `||`, `&&` and `!` which accept arguments of any type. For these operators, all values except `false` and `nil` will evaluate to true: - -```iex -# or -iex> 1 || true -1 -iex> false || 11 -11 - -# and -iex> nil && 13 -nil -iex> true && 17 -17 - -# ! -iex> !true -false -iex> !1 -false -iex> !nil -true -``` - -As a rule of thumb, use `and`, `or` and `not` when you are expecting booleans. If any of the arguments are non-boolean, use `&&`, `||` and `!`. - -Elixir also provides `==`, `!=`, `===`, `!==`, `<=`, `>=`, `<` and `>` as comparison operators: - -```iex -iex> 1 == 1 -true -iex> 1 != 2 -true -iex> 1 < 2 -true -``` - -The difference between `==` and `===` is that the latter is more strict when comparing integers and floats: - -```iex -iex> 1 == 1.0 -true -iex> 1 === 1.0 -false -``` - -In Elixir, we can compare two different data types: - -```iex -iex> 1 < :atom -true -``` - -The reason we can compare different data types is pragmatism. Sorting algorithms don't need to worry about different data types in order to sort. The overall sorting order is defined below: - - number < atom < reference < functions < port < pid < tuple < maps < list < bitstring - -You don't actually need to memorize this ordering, but it is important just to know an order exists. - -Well, that is it for the introduction. In the next chapter, we are going to discuss some basic functions, data type conversions and a bit of control-flow. diff --git a/getting_started/4.markdown b/getting_started/4.markdown deleted file mode 100644 index 529ef761d..000000000 --- a/getting_started/4.markdown +++ /dev/null @@ -1,177 +0,0 @@ ---- -layout: getting_started -title: 4 Pattern matching -guide: 4 ---- - -# {{ page.title }} - -
      - -In this chapter, we will show how the `=` operator in Elixir is actually a match operator and how to use it to pattern match inside data structures. Finally, we will learn about the pin operator `^` used to access previously bound values. - -## 4.1 The match operator - -We have used the `=` operator a couple times to assign variables in Elixir: - -```iex -iex> x = 1 -1 -iex> x -1 -``` - -In Elixir, the `=` operator is actually called *the match operator*. Let's see why: - -```iex -iex> 1 = x -1 -iex> 2 = x -** (MatchError) no match of right hand side value: 1 -``` - -Notice that `1 = x` is a valid expression, and it matched because both the left and right side are equal to 1. When the sides do not match, a `MatchError` is raised. - -A variable can only be assigned on the left side of `=`: - -```iex -iex> 1 = unknown -** (RuntimeError) undefined function: unknown/0 -```` - -Since there is no variable `unknown` previously defined, Elixir imagined you were trying to call a function named `unknown/0`, but such a function does not exist. - -## 4.2 Pattern matching - -The match operator is not only used to match against simple values, but it is also useful for destructuring more complex data types. For example, we can pattern match on tuples: - -```iex -iex> {a, b, c} = {:hello, "world", 42} -{:hello, "world", 42} -iex> a -:hello -iex> b -"world" -``` - -A pattern match will error in the case that the sides can't match. This is, for example, the case when the tuples have different sizes: - -```iex -iex> {a, b, c} = {:hello, "world"} -** (MatchError) no match of right hand side value: {:hello, "world"} -``` - -And also when comparing different types: - -```iex -iex> {a, b, c} = [:hello, "world", "!"] -** (MatchError) no match of right hand side value: [:hello, "world", "!"] -``` - -More interestingly, we can match on specific values. The example below asserts that the left side will only match the right side when the right side is a tuple that starts with the atom `:ok`: - -```iex -iex> {:ok, result} = {:ok, 13} -{:ok, 13} -iex> result -13 - -iex> {:ok, result} = {:error, :oops} -** (MatchError) no match of right hand side value: {:error, :oops} -``` - -We can pattern match on lists: - -```iex -iex> [a, b, c] = [1, 2, 3] -[1, 2, 3] -iex> a -1 -``` - -A list also supports matching on its own head and tail: - -```iex -iex> [head | tail] = [1, 2, 3] -[1, 2, 3] -iex> head -1 -iex> tail -[2, 3] -``` - -Similar to the `hd/1` and `tl/1` functions, we can't match an empty list with a head and tail pattern: - -```iex -iex> [h|t] = [] -** (MatchError) no match of right hand side value: [] -``` - -The `[head | tail]` format is not only used on pattern matching but also for prepending items to a list: - -```iex -iex> list = [1, 2, 3] -[1, 2, 3] -iex> [0|list] -[0, 1, 2, 3] -``` - -Pattern matching allows developers to easily destructure data types such as tuples and lists. As we will see in following chapters, it is one of the foundations of recursion in Elixir and applies to other types as well, like maps and binaries. - -## 4.3 The pin operator - -Variables in Elixir can be rebound: - -```iex -iex> x = 1 -1 -iex> x = 2 -2 -``` - -The pin operator `^` can be used when there is no interest in rebinding a variable but rather in matching against its value prior to the match: - -```iex -iex> x = 1 -1 -iex> ^x = 2 -** (MatchError) no match of right hand side value: 2 -iex> {x, ^x} = {2, 1} -{2, 1} -iex> x -2 -``` - -Notice that if a variable is mentioned more than once in a pattern, all references should bind to the same pattern: - -```iex -iex> {x, x} = {1, 1} -1 -iex> {x, x} = {1, 2} -** (MatchError) no match of right hand side value: {1, 2} -``` - -In some cases, you don't care about a particular value in a pattern. It is a common practice to bind those values to the underscore, `_`. For example, if only the head of the list matters to us, we can assign the tail to underscore: - -```iex -iex> [h | _] = [1, 2, 3] -[1, 2, 3] -iex> h -1 -``` - -The variable `_` is special in that it can never be read from. Trying to read from it gives an unbound variable error: - -```iex -iex> _ -** (CompileError) iex:1: unbound variable _ -``` - -Although pattern matching allows us to build powerful constructs, its usage is limited. For instance, you cannot make function calls on the left side of a match. The following example is invalid: - -```iex -iex> length([1,[2],3]) = 3 -** (CompileError) iex:1: illegal pattern -``` - -This finishes our introduction to pattern matching. As we will see in the next chapter, pattern matching is very common in many language constructs. diff --git a/getting_started/5.markdown b/getting_started/5.markdown deleted file mode 100644 index 709c0db63..000000000 --- a/getting_started/5.markdown +++ /dev/null @@ -1,263 +0,0 @@ ---- -layout: getting_started -title: 5 case, cond and if -guide: 5 ---- - -# {{ page.title }} - -
      - -In this chapter, we will learn about the `case`, `cond` and `if` control-flow structures. - -## 5.1 case - -`case` allows us to compare a value against many patterns until we find a matching one: - -```iex -iex> case {1, 2, 3} do -...> {4, 5, 6} -> -...> "This clause won't match" -...> {1, x, 3} -> -...> "This clause will match and bind x to 2 in this clause" -...> _ -> -...> "This clause would match any value" -...> end -``` - -If you want to pattern match against an existing variable, you need to use the `^` operator: - -```iex -iex> x = 1 -1 -iex> case 10 do -...> ^x -> "Won't match" -...> _ -> "Will match" -...> end -``` - -Clauses also allow extra conditions to be specified via guards: - -```iex -iex> case {1, 2, 3} do -...> {1, x, 3} when x > 0 -> -...> "Will match" -...> _ -> -...> "Won't match" -...> end -``` - -The first clause above will only match when `x` is positive. - -## 5.2 Expressions in guard clauses. - -The Erlang VM only allows a limited set of expressions in guards: - -* comparison operators (`==`, `!=`, `===`, `!==`, `>`, `<`, `<=`, `>=`) -* boolean operators (`and`, `or`) and negation operators (`not`, `!`) -* arithmetic operators (`+`, `-`, `*`, `/`) -* `<>` and `++` as long as the left side is a literal -* the `in` operator -* all the following type check functions: - - * `is_atom/1` - * `is_binary/1` - * `is_bitstring/1` - * `is_boolean/1` - * `is_float/1` - * `is_function/1` - * `is_function/2` - * `is_integer/1` - * `is_list/1` - * `is_map/1` - * `is_number/1` - * `is_pid/1` - * `is_port/1` - * `is_reference/1` - * `is_tuple/1` - -* plus these functions: - - * `abs(number)` - * `bit_size(bitstring)` - * `byte_size(bitstring)` - * `div(integer, integer)` - * `elem(tuple, n)` - * `hd(list)` - * `length(list)` - * `map_size(map)` - * `node()` - * `node(pid | ref | port)` - * `rem(integer, integer)` - * `round(number)` - * `self()` - * `tl(list)` - * `trunc(number)` - * `tuple_size(tuple)` - -Keep in mind errors in guards do not leak but simply make the guard fail: - -```iex -iex> hd(1) -** (ArgumentError) argument error - :erlang.hd(1) -iex> case 1 do -...> x when hd(x) -> "Won't match" -...> x -> "Got: #{x}" -...> end -"Got 1" -``` - -If none of the clauses match, an error is raised: - -```iex -iex> case :ok do -...> :error -> "Won't match" -...> end -** (CaseClauseError) no case clause matching: :ok -``` - -Note anonymous functions can also have multiple clauses and guards: - -```elixir -iex> f = fn -...> x, y when x > 0 -> x + y -...> x, y -> x * y -...> end -#Function<12.71889879/2 in :erl_eval.expr/5> -iex> f.(1, 3) -4 -iex> f.(-1, 3) --3 -``` - -The number of arguments in each anonymous function clause needs to be the same, otherwise an error is raised. - -## 5.3 cond - -`case` is useful when you need to match against different values. However, in many circumstances, we want to check different conditions and find the first one that evaluates to true. In such cases, one may use `cond`: - -```iex -iex> cond do -...> 2 + 2 == 5 -> -...> "This will not be true" -...> 2 * 2 == 3 -> -...> "Nor this" -...> 1 + 1 == 2 -> -...> "But this will" -...> end -"But this will" -``` - -This is equivalent to `else if` clauses in many imperative languages (although used way less frequently here). - -If none of the conditions return true, an error is raised. For this reason, it may be necessary to add a final condition, equal to `true`, which will always match: - -```iex -iex> cond do -...> 2 + 2 == 5 -> -...> "This is never true" -...> 2 * 2 == 3 -> -...> "Nor this" -...> true -> -...> "This is always true (equivalent to else)" -...> end -``` - -Finally, note `cond` considers any value besides `nil` and `false` to be true: - -```iex -iex> cond do -...> hd([1,2,3]) -> -...> "1 is considered as true" -...> end -"1 is considered as true" -``` - -## 5.4 if and unless - -Besides `case` and `cond`, Elixir also provides the macros `if/2` and `unless/2` which are useful when you need to check for just one condition: - -```iex -iex> if true do -...> "This works!" -...> end -"This works!" -iex> unless true do -...> "This will never be seen" -...> end -nil -``` - -If the condition given to `if/2` returns `false` or `nil`, the body given between `do/end` is not executed and it simply returns `nil`. The opposite happens with `unless/2`. - -They also support `else` blocks: - -```iex -iex> if nil do -...> "This won't be seen" -...> else -...> "This will" -...> end -"This will" -``` - -> Note: An interesting note regarding `if/2` and `unless/2` is that they are implemented as macros in the language; they aren't special language constructs as they would be in many languages. You can check the documentation and the source of `if/2` in [the `Kernel` module docs](/docs/stable/elixir/Kernel.html). The `Kernel` module is also where operators like `+/2` and functions like `is_function/2` are defined, all automatically imported and available in your code by default. - -## 5.5 `do` blocks - -At this point, we have learned four control structures: `case`, `cond`, `if` and `unless`, and they were all wrapped in `do`/`end` blocks. It happens we could also write `if` as follows: - -```iex -iex> if true, do: 1 + 2 -3 -``` - -In Elixir, `do`/`end` blocks are a convenience for passing a group of expressions to `do:`. These are equivalent: - -```iex -iex> if true do -...> a = 1 + 2 -...> a + 10 -...> end -13 -iex> if true, do: ( -...> a = 1 + 2 -...> a + 10 -...> ) -13 -``` - -We say the second syntax is using **keyword lists**. We can pass `else` using this syntax: - -```iex -iex> if false, do: :this, else: :that -:that -``` - -One thing to keep in mind when using `do`/`end` blocks is they are always bound to the outermost function call. For example, the following expression: - -```iex -iex> is_number if true do -...> 1 + 2 -...> end -``` - -Would be parsed as: - -```iex -iex> is_number(if true) do -...> 1 + 2 -...> end -``` - -Which leads to an undefined function error as Elixir attempts to invoke `is_number/2`. Adding explicit parentheses is enough to resolve the ambiguity: - -```iex -iex> is_number(if true do -...> 1 + 2 -...> end) -true -``` - -Keyword lists play an important role in the language and are quite common in many functions and macros. We will explore them a bit more in a future chapter. Now it is time to talk about "Binaries, strings and char lists". diff --git a/getting_started/6.markdown b/getting_started/6.markdown deleted file mode 100644 index 1f476a4b1..000000000 --- a/getting_started/6.markdown +++ /dev/null @@ -1,186 +0,0 @@ ---- -layout: getting_started -title: 6 Binaries, strings and char lists -guide: 6 ---- - -# {{ page.title }} - -
      - -In "Basic types", we learned about strings and used the `is_binary/1` function for checks: - -```iex -iex> string = "hello" -"hello" -iex> is_binary string -true -``` - -In this chapter, we will understand what binaries are, how they associate with strings, and what a single-quoted value, `'like this'`, means in Elixir. - -## 6.1 UTF-8 and Unicode - -A string is a UTF-8 encoded binary. In order to understand exactly what we mean by that, we need to understand the difference between bytes and code points. - -The Unicode standard assigns code points to many of the characters we know. For example, the letter `a` has code point `97` while the letter `ł` has code point `322`. When writing the string `"hełło"` to disk, we need to convert this code point to bytes. If we adopted a rule that said one byte represents one code point, we wouldn't be able to write `"hełło"`, because it uses the code point `322` for `ł`, and one byte can only represent a number from `0` to `255`. But of course, given you can actually read `"hełło"` on your screen, it must be represented *somehow*. That's where encodings come in. - -When representing code points in bytes, we need to encode them somehow. Elixir chose the UTF-8 encoding as its main and default encoding. When we say a string is a UTF-8 encoded binary, we mean a string is a bunch of bytes organized in a way to represent certain code points, as specified by the UTF-8 encoding. - -Since we have code points like `ł` assigned to the number `322`, we actually need more than one byte to represent it. That's why we see a difference when we calculate the `byte_size/1` of a string compared to its `String.length/1`: - -```iex -iex> string = "hełło" -"hełło" -iex> byte_size string -7 -iex> String.length string -5 -``` - -UTF-8 requires one byte to represent the code points `h`, `e` and `o`, but two bytes to represent `ł`. In Elixir, you can get a code point's value by using `?`: - -```iex -iex> ?a -97 -iex> ?ł -322 -``` - -You can also use the functions in [the `String` module](/docs/stable/elixir/String.html) to split a string in its code points: - -```iex -iex> String.codepoints("hełło") -["h", "e", "ł", "ł", "o"] -``` - -You will see that Elixir has excellent support for working with strings. It also supports many of the Unicode operations. In fact, Elixir passes all the tests showcased in the article ["The string type is broken"](http://mortoray.com/2013/11/27/the-string-type-is-broken/). - -However, strings are just part of the story. If a string is a binary, and we have used the `is_binary/1` function, Elixir must have an underlying type empowering strings. And it does. Let's talk about binaries! - -## 6.2 Binaries (and bitstrings) - -In Elixir, you can define a binary using `<<>>`: - -```iex -iex> <<0, 1, 2, 3>> -<<0, 1, 2, 3>> -iex> byte_size <<0, 1, 2, 3>> -4 -``` - -A binary is just a sequence of bytes. Of course, those bytes can be organized in any way, even in a sequence that does not make them a valid string: - -```iex -iex> String.valid?(<<239, 191, 191>>) -false -``` - -The string concatenation operation is actually a binary concatenation operator: - -```iex -iex> <<0, 1>> <> <<2, 3>> -<<0, 1, 2, 3>> -``` - -A common trick in Elixir is to concatenate the null byte `<<0>>` to a string to see its inner binary representation: - -```iex -iex> "hełło" <> <<0>> -<<104, 101, 197, 130, 197, 130, 111, 0>> -``` - -Each number given to a binary is meant to represent a byte and therefore must go up to 255. Binaries allow modifiers to be given to store numbers bigger than 255 or to convert a code point to its utf8 representation: - -```iex -iex> <<255>> -<<255>> -iex> <<256>> # truncated -<<0>> -iex> <<256 :: size(16)>> # use 16 bits (2 bytes) to store the number -<<1, 0>> -iex> <<256 :: utf8>> # the number is a code point -"Ā" -iex> <<256 :: utf8, 0>> -<<196, 128, 0>> -``` - -If a byte has 8 bits, what happens if we pass a size of 1 bit? - -```iex -iex> <<1 :: size(1)>> -<<1::size(1)>> -iex> <<2 :: size(1)>> # truncated -<<0::size(1)>> -iex> is_binary(<< 1 :: size(1)>>) -false -iex> is_bitstring(<< 1 :: size(1)>>) -true -iex> bit_size(<< 1 :: size(1)>>) -1 -``` - -The value is no longer a binary, but a bitstring -- just a bunch of bits! So a binary is a bitstring where the number of bits is divisible by 8! - -We can also pattern match on binaries / bitstrings: - -```iex -iex> <<0, 1, x>> = <<0, 1, 2>> -<<0, 1, 2>> -iex> x -2 -iex> <<0, 1, x>> = <<0, 1, 2, 3>> -** (MatchError) no match of right hand side value: <<0, 1, 2, 3>> -``` - -Note each entry in the binary is expected to match exactly 8 bits. However, we can match on the rest of the binary modifier: - -```iex -iex> <<0, 1, x :: binary>> = <<0, 1, 2, 3>> -<<0, 1, 2, 3>> -iex> x -<<2, 3>> -``` - -The pattern above only works if the binary is at the end of `<<>>`. Similar results can be achieved with the string concatenation operator `<>`: - -```iex -iex> "he" <> rest = "hello" -"hello" -iex> rest -"llo" -``` - -This finishes our tour of bitstrings, binaries and strings. A string is a UTF-8 encoded binary, and a binary is a bitstring where the number of bits is divisible by 8. Although this shows the flexibility Elixir provides to work with bits and bytes, 99% of the time you will be working with binaries and using the `is_binary/1` and `byte_size/1` functions. - -## 6.3 Char lists - -A char list is nothing more than a list of characters: - -```iex -iex> 'hełło' -[104, 101, 322, 322, 111] -iex> is_list 'hełło' -true -iex> 'hello' -'hello' -``` - -You can see that, instead of containing bytes, a char list contains the code points of the characters between single-quotes (note that iex will only output code points if any of the chars is outside the ASCII range). So while double-quotes represent a string (i.e. a binary), single-quotes represents a char list (i.e. a list). - -In practice, char lists are used mostly when interfacing with Erlang, in particular old libraries that do not accept binaries as arguments. You can convert a char list to a string and back by using the `to_string/1` and `to_char_list/1` functions: - -```iex -iex> to_char_list "hełło" -[104, 101, 322, 322, 111] -iex> to_string 'hełło' -"hełło" -iex> to_string :hello -"hello" -iex> to_string 1 -"1" -``` - -Note that those functions are polymorphic. They not only convert char lists to strings, but also integers to strings, atoms to strings, and so on. - -With binaries, strings, and char lists out of the way, it is time to talk about key-value data structures. diff --git a/getting_started/7.markdown b/getting_started/7.markdown deleted file mode 100644 index 5c462a919..000000000 --- a/getting_started/7.markdown +++ /dev/null @@ -1,186 +0,0 @@ ---- -layout: getting_started -title: 7 Keywords, maps and dicts -guide: 7 ---- - -# {{ page.title }} - -
      - -So far we haven't discussed any associative data structures, i.e. data structures that are able to associate a certain value (or multiple values) to a key. Different languages call these different names like dictionaries, hashes, associative arrays, maps, etc. - -In Elixir, we have two main associative data structures: keyword lists and maps. It's time to learn more about them! - -## 7.1 Keyword lists - -In many functional programming languages, it is common to use a list of 2-item tuples as the representation of an associative data structure. In Elixir, when we have a list of tuples and the first item of the tuple (i.e. the key) is an atom, we call it a keyword list: - -```iex -iex> list = [{:a, 1}, {:b, 2}] -[a: 1, b: 2] -iex> list == [a: 1, b: 2] -true -iex> list[:a] -1 -``` - -As you can see above, Elixir supports a special syntax for defining such lists, and underneath they just map to a list of tuples. Since they are simply lists, all operations available to lists, including their performance characteristics, also apply to keyword lists. - -For example, we can use `++` to add new values to a keyword list: - -```iex -iex> list ++ [c: 3] -[a: 1, b: 2, c: 3] -iex> [a: 0] ++ list -[a: 0, a: 1, b: 2] -``` - -Note that values added to the front are the ones fetched on lookup: - -```iex -iex> new_list = [a: 0] ++ list -[a: 0, a: 1, b: 2] -iex> new_list[:a] -0 -``` - -Keyword lists are important because they have two special characteristics: - -* They keep the keys ordered, as specified by the developer. -* They allow a key to be given more than once. - -For example, [the Ecto library](https://github.com/elixir-lang/ecto) makes use of both features to provide an elegant DSL for writing database queries: - -```elixir -query = from w in Weather, - where: w.prcp > 0, - where: w.temp < 20, - select: w -``` - -Those features are what prompted keyword lists to be the default mechanism for passing options to functions in Elixir. In chapter 5, when we discussed the `if/2` macro, we mentioned the following syntax is supported: - -```iex -iex> if false, do: :this, else: :that -:that -``` - -The `do:` and `else:` pairs are keyword lists! In fact, the call above is equivalent to: - -```iex -iex> if(false, [do: :this, else: :that]) -:that -``` - -In general, when the keyword list is the last argument of a function, the square brackets are optional. - -In order to manipulate keyword lists, Elixir provides [the `Keyword` module](/docs/stable/elixir/Keyword.html). Remember though keyword lists are simply lists, and as such they provide the same linear performance characteristics as lists. The longer the list, the longer it will take to find a key, to count the number of items, and so on. For this reason, keyword lists are used in Elixir mainly as options. If you need to store many items or guarantee one-key associates with at maximum one-value, you should use maps instead. - -Note we can also pattern match on keyword lists: - -```iex -iex> [a: a] = [a: 1] -[a: 1] -iex> a -1 -iex> [a: a] = [a: 1, b: 2] -** (MatchError) no match of right hand side value: [a: 1, b: 2] -iex> [b: b, a: a] = [a: 1, b: 2] -** (MatchError) no match of right hand side value: [a: 1, b: 2] -``` - -However this is rarely done in practice since pattern matching on lists require the number of items and their order to match. - -## 7.2 Maps - -Whenever you need a key-value store, maps are the "go to" data structure in Elixir. A map is created using the `%{}` syntax: - -```iex -iex> map = %{:a => 1, 2 => :b} -%{2 => :b, :a => 1} -iex> map[:a] -1 -iex> map[2] -:b -``` - -Compared to keyword lists, we can already see two differences: - -* Maps allow any value as a key. -* Maps' keys do not follow any ordering. - -If you pass duplicate keys when creating a map, the last one wins: - -```iex -iex> %{1 => 1, 1 => 2} -%{1 => 2} -``` - -When all the keys in a map are atoms, you can use the keyword syntax for convenience: - -```iex -iex> map = %{a: 1, b: 2} -%{a: 1, b: 2} -``` - -In contrast to keyword lists, maps are very useful with pattern matching: - -```iex -iex> %{} = %{:a => 1, 2 => :b} -%{:a => 1, 2 => :b} -iex> %{:a => a} = %{:a => 1, 2 => :b} -%{:a => 1, 2 => :b} -iex> a -1 -iex> %{:c => c} = %{:a => 1, 2 => :b} -** (MatchError) no match of right hand side value: %{2 => :b, :a => 1} -``` - -As shown above, a map matches as long as the given keys exist in the given map. Therefore, an empty map matches all maps. - -One interesting property about maps is that they provide a particular syntax for updating and accessing atom keys: - -```iex -iex> map = %{:a => 1, 2 => :b} -%{:a => 1, 2 => :b} -iex> map.a -1 -iex> %{map | :a => 2} -%{:a => 2, 2 => :b} -iex> %{map | :c => 3} -** (ArgumentError) argument error -``` - -Both access and update syntaxes above require the given keys to exist. For example, the last line failed because there is no `:c` in the map. This is very useful when you are working with maps where you only expect certain keys to exist. - -In future chapters, we will also learn about structs, which provide compile-time guarantees and the foundation for polymorphism in Elixir. Structs are built on top of maps where the update guarantees above are proven to be quite useful. - -Manipulating maps is done via [the `Map` module](/docs/stable/elixir/Map.html), it provides a very similar API to the `Keyword` module. This is because both modules implement the `Dict` behaviour. - -> Note: Maps were recently introduced into the Erlang VM with [EEP 43](http://www.erlang.org/eeps/eep-0043.html). Erlang 17 provides a partial implementation of the EEP, where only "small maps" are supported. This means maps have good performance characteristics only when storing at maximum a couple of dozens keys. To fill in this gap, Elixir also provides [the `HashDict` module](/docs/stable/elixir/HashDict.html) which uses a hashing algorithm to provide a dictionary that supports hundreds of thousands keys with good performance. - -## 7.3 Dicts - -In Elixir, both keyword lists and maps are called dictionaries. In other words, a dictionary is like an interface (we call them behaviours in Elixir) and both keyword lists and maps modules implement this interface. - -This interface is defined in the [the `Dict` module](/docs/stable/elixir/Dict.html) module which also provides an API that delegates to the underlying implementations: - -```iex -iex> keyword = [] -[] -iex> map = %{} -%{} -iex> Dict.put(keyword, :a, 1) -[a: 1] -iex> Dict.put(map, :a, 1) -%{a: 1} -``` - -The `Dict` module allows any developer to implement their own variation of `Dict`, with specific characteristics, and hook into existing Elixir code. The `Dict` module also provides functions that are meant to work across dictionaries. For example, `Dict.equal?/2` can compare two dictionaries of any kind. - -That said, you may be wondering, which of `Keyword`, `Map` or `Dict` modules should you use in your code? The answer is: it depends. - -If your code is expecting a keyword as an argument, explicitly use the `Keyword` module. If you want to manipulate a map, use the `Map` module. However, if your API is meant to work with any dictionary, use the `Dict` module (and make sure to write tests that pass different dict implementations as arguments). - -This concludes our introduction to associative data structures in Elixir. You will find out that given keyword lists and maps, you will always have the right tool to tackle problems that require associative data structures in Elixir. diff --git a/getting_started/8.markdown b/getting_started/8.markdown deleted file mode 100644 index 956f332b6..000000000 --- a/getting_started/8.markdown +++ /dev/null @@ -1,251 +0,0 @@ ---- -layout: getting_started -title: 8 Modules -guide: 8 ---- - -# {{ page.title }} - -
      - -In Elixir we group several functions into modules. We've already used many different modules in the previous chapters like [the `String` module](/docs/stable/elixir/String.html): - -```iex -iex> String.length "hello" -5 -``` - -In order to create our own modules in Elixir, we use the `defmodule` macro. We use the `def` macro to define functions in that module: - -```iex -iex> defmodule Math do -...> def sum(a, b) do -...> a + b -...> end -...> end - -iex> Math.sum(1, 2) -3 -``` - -In the following sections, our examples are going to get a bit more complex, and it can be tricky to type them all in the shell. It's about time for us to learn how to compile Elixir code and also how to run Elixir scripts. - -## 8.1 Compilation - -Most of the time it is convenient to write modules into files so they can be compiled and reused. Let's assume we have a file named `math.ex` with the following contents: - -```elixir -defmodule Math do - def sum(a, b) do - a + b - end -end -``` - -This file can be compiled using `elixirc`: - - $ elixirc math.ex - -This will generate a file named `Elixir.Math.beam` containing the bytecode for the defined module. If we start `iex` again, our module definition will be available (provided that `iex` is started in the same directory the bytecode file is in): - -```iex -iex> Math.sum(1, 2) -3 -``` - -Elixir projects are usually organized into three directories: - -* ebin - contains the compiled bytecode -* lib - contains elixir code (usually `.ex` files) -* test - contains tests (usually `.exs` files) - -When working on actual projects, the build tool called `mix` will be responsible for compiling and setting up the proper paths for you. For learning purposes, Elixir also supports a scripted mode which is more flexible and does not generate any compiled artifacts. - -## 8.2 Scripted mode - -In addition to the Elixir file extension `.ex`, Elixir also supports `.exs` files for scripting. Elixir treats both files exactly the same way, the only difference is in intention. `.ex` files are meant to be compiled while `.exs` files are used for scripting, without the need for compilation. For instance, we can create a file called `math.exs`: - -```elixir -defmodule Math do - def sum(a, b) do - a + b - end -end - -IO.puts Math.sum(1, 2) -``` - -And execute it as: - - $ elixir math.exs - -The file will be compiled in memory and executed, printing "3" as the result. No bytecode file will be created. In the following examples, we recommend you write your code into script files and execute them as shown above. - -## 8.3 Named functions - -Inside a module, we can define functions with `def/2` and private functions with `defp/2`. A function defined with `def/2` can be invoked from other modules while a private function can only be invoked locally. - -```elixir -defmodule Math do - def sum(a, b) do - do_sum(a, b) - end - - defp do_sum(a, b) do - a + b - end -end - -Math.sum(1, 2) #=> 3 -Math.do_sum(1, 2) #=> ** (UndefinedFunctionError) -``` - -Function declarations also support guards and multiple clauses. If a function has several clauses, Elixir will try each clause until it finds one that matches. Here is an implementation of a function that checks if the given number is zero or not: - -```elixir -defmodule Math do - def zero?(0) do - true - end - - def zero?(x) when is_number(x) do - false - end -end - -Math.zero?(0) #=> true -Math.zero?(1) #=> false - -Math.zero?([1,2,3]) -#=> ** (FunctionClauseError) -``` - -Giving an argument that does not match any of the clauses raises an error. - -## 8.4 Function capturing - -Throughout this tutorial, we have been using the notation `name/arity` to refer to functions. It happens that this notation can actually be used to retrieve a named function as a function type. Let's start `iex` and run the `math.exs` file defined above: - - $ iex math.exs - -```iex -iex> Math.zero?(0) -true -iex> fun = &Math.zero?/1 -&Math.zero?/1 -iex> is_function fun -true -iex> fun.(0) -true -```` - -Local or imported functions, like `is_function/1`, can be captured without the module: - -```iex -iex> &is_function/1 -&:erlang.is_function/1 -iex> (&is_function/1).(fun) -true -``` - -Note the capture syntax can also be used as a shortcut for creating functions: - -```iex -iex> fun = &(&1 + 1) -#Function<6.71889879/1 in :erl_eval.expr/5> -iex> fun.(1) -2 -``` - -The `&1` represents the first argument passed into the function. `&(&1+1)` above is exactly the same as `fn x -> x + 1 end`. The syntax above is useful for short function definitions. You can read more about the capture operator `&` in [the `Kernel.SpecialForms` documentation](/docs/stable/elixir/Kernel.SpecialForms.html). - -## 8.5 Default arguments - -Named functions in Elixir also support default arguments: - -```elixir -defmodule Concat do - def join(a, b, sep \\ " ") do - a <> sep <> b - end -end - -IO.puts Concat.join("Hello", "world") #=> Hello world -IO.puts Concat.join("Hello", "world", "_") #=> Hello_world -``` - -Any expression is allowed to serve as a default value, but it won't be evaluated during the function definition; it will simply be stored for later use. Every time the function is invoked and any of its default values have to be used, the expression for that default value will be evaluated: - -```elixir -defmodule DefaultTest do - def dowork(x \\ IO.puts "hello") do - x - end -end -``` - -```iex -iex> DefaultTest.dowork 123 -123 -iex> DefaultTest.dowork -hello -:ok -``` - -If a function with default values has multiple clauses, it is recommended to create a function head (without an actual body), just for declaring defaults: - -```elixir -defmodule Concat do - def join(a, b \\ nil, sep \\ " ") - - def join(a, b, _sep) when is_nil(b) do - a - end - - def join(a, b, sep) do - a <> sep <> b - end -end - -IO.puts Concat.join("Hello", "world") #=> Hello world -IO.puts Concat.join("Hello", "world", "_") #=> Hello_world -IO.puts Concat.join("Hello") #=> Hello -``` - -When using default values, one must be careful to avoid overlapping function definitions. Consider the following example: - -```elixir -defmodule Concat do - def join(a, b) do - IO.puts "***First join" - a <> b - end - - def join(a, b, sep \\ " ") do - IO.puts "***Second join" - a <> sep <> b - end -end -``` - -If we save the code above in a file named "concat.ex" and compile it, Elixir will emit the following warning: - - concat.ex:7: this clause cannot match because a previous clause at line 2 always matches - -The compiler is telling us that invoking the `join` function with two arguments will always choose the first definition of `join` whereas the second one will only be invoked when three arguments are passed: - - $ iex concat.exs - -```iex -iex> Concat.join "Hello", "world" -***First join -"Helloworld" -``` - -```iex -iex> Concat.join "Hello", "world", "_" -***Second join -"Hello_world" -``` - -This finishes our short introduction to modules. In the next chapters, we will learn how to use named functions for recursion, explore Elixir lexical directives that can be used for importing functions from other modules and discuss module attributes. diff --git a/getting_started/9.markdown b/getting_started/9.markdown deleted file mode 100644 index c8a0d3861..000000000 --- a/getting_started/9.markdown +++ /dev/null @@ -1,120 +0,0 @@ ---- -layout: getting_started -title: 9 Recursion -guide: 9 ---- - -# {{ page.title }} - -
      - -Due to immutability, loops in Elixir (and in functional programming languages) are written differently from conventional imperative languages. For example, in an imperative language, one would write: - -```c -for(i = 0; i < array.length; i++) { - array[i] = array[i] * 2 -} -``` - -In the example above, we are mutating the array and the helper variable `i`. That's not possible in Elixir. Instead, functional languages rely on recursion: a function is called recursively until a condition is reached that stops the recursive action from continuing. Consider the example below that prints a string an arbitrary amount of times: - -```elixir -defmodule Recursion do - def print_multiple_times(msg, n) when n <= 1 do - IO.puts msg - end - - def print_multiple_times(msg, n) do - IO.puts msg - print_multiple_times(msg, n - 1) - end -end - -Recursion.print_multiple_times("Hello!", 3) -# Hello! -# Hello! -# Hello! -``` - -Similar to case, a function may have many clauses. A particular clause is executed when the arguments passed to the function match the clause's argument patterns and its guard evaluates to `true`. - -Above when `print_multiple_times/2` is initially called, the argument `n` is equal to `3`. - -The first clause has a guard which says use this definition if and only if `n` is less than or equal to `1`. Since this is not the case, Elixir proceeds to the next clause's definition. - -The second definition matches the pattern and has no guard so it will be executed. It first prints our `msg` and then calls itself passing `n - 1` (`2`) as the second argument. Our `msg` is printed and `print_multiple_times/2` is called again this time with the second argument set to `1`. - -Because `n` is now set to `1`, the guard to our first definition of `print_multiple_times/2` evaluates to true, and we execute this particular definition. The `msg` is printed, and there is nothing left to execute. - -We defined `print_multiple_times/2` so that no matter what number is passed as the second argument it either triggers our first definition (known as a "base case") or it triggers our second definition which will ensure that we get exactly 1 step closer to our base case. - -Let's now see how we can use the power of recursion to sum a list of numbers: - -```elixir -defmodule Math do - def sum_list([head|tail], accumulator) do - sum_list(tail, head + accumulator) - end - - def sum_list([], accumulator) do - accumulator - end -end - -Math.sum_list([1, 2, 3], 0) #=> 6 -``` - -We invoke `sum_list` with a list `[1,2,3]` and the initial value `0` as arguments. We will try each clause until we find one that matches according to the pattern matching rules. In this case, the list `[1,2,3]` matches against `[head|tail]` which assigns `head = 1` and `tail = [2,3]` while `accumulator` is set to `0`. - -Then, we add the head of the list to the accumulator `head + accumulator` and call `sum_list` again, recursively, passing the tail of the list as its first argument. The tail will once again match `[head|tail]` until the list is empty, as seen below: - -```elixir -sum_list [1, 2, 3], 0 -sum_list [2, 3], 1 -sum_list [3], 3 -sum_list [], 6 -``` - -When the list is empty, it will match the final clause which returns the final result of `6`. - -The process of taking a list and "reducing" it down to one value is known as a "reduce" algorithm and is central to functional programming. - -What if we instead want to double all of the values in our list? - -```elixir -defmodule Math do - def double_each([head|tail]) do - [head * 2| double_each(tail)] - end - - def double_each([]) do - [] - end -end - -Math.double_each([1, 2, 3]) #=> [2, 4, 6] -``` - -Here we have used recursion to traverse a list doubling each element and returning a new list. The process of taking a list and "mapping" over it is known as a "map" algorithm. - -Recursion and tail call optimization are an important part of Elixir and are commonly used to create loops. However, when programming Elixir you will rarely use recursion as above to manipulate lists. - -The [`Enum` module](/docs/stable/elixir/Enum.html), which we are going to study in the next chapter, already provides many conveniences for working with lists. For instance, the examples above could be written as: - -```iex -iex> Enum.reduce([1, 2, 3], 0, fn(x, acc) -> x + acc end) -6 -iex> Enum.map([1, 2, 3], fn(x) -> x * 2 end) -[2, 4, 6] -``` - -Or, using the capture syntax: - -```iex -iex> Enum.reduce([1, 2, 3], 0, &+/2) -6 -iex> Enum.map([1, 2, 3], &(&1 * 2)) -[2, 4, 6] -``` - -So let's take a deeper look at Enumerables and Streams. diff --git a/getting_started/meta/1.markdown b/getting_started/meta/1.markdown deleted file mode 100644 index 57c816267..000000000 --- a/getting_started/meta/1.markdown +++ /dev/null @@ -1,148 +0,0 @@ ---- -layout: getting_started -title: 1 Quote and unquote -guide: 1 ---- - -# {{ page.title }} - -
      - -An Elixir program can be represented by its own data structures. In this chapter, we will learn what those structures look like and how to compose them. The concepts learned in this chapter are the building blocks for macros, which we are going to take a deeper look at in the next chapter. - -## 1.1 Quoting - -The building block of an Elixir program is a tuple with three elements. For example, the function call `sum(1,2,3)` is represented internally as: - -```elixir -{:sum, [], [1, 2, 3]} -``` - -You can get the representation of any expression by using the `quote` macro: - -```iex -iex> quote do: sum(1, 2, 3) -{:sum, [], [1, 2, 3]} -``` - -The first element is the function name, the second is a keyword list containing metadata and the third is the arguments list. - -Operators are also represented as such tuples: - -```iex -iex> quote do: 1 + 2 -{:+, [context: Elixir, import: Kernel], [1, 2]} -``` - -Even a map is represented as a call to `%{}`: - -```iex -iex> quote do: %{1 => 2} -{:%{}, [], [{1, 2}]} -``` - -Variables are also represented using such triplets, except the last element is an atom, instead of a list: - -```iex -iex> quote do: x -{ :x, [], Elixir } -``` - -When quoting more complex expressions, we can see that the code is represented in such tuples, which are often nested inside each other in a structure resembling a tree. Many languages would call such representations an Abstract Syntax Tree (AST). Elixir calls them quoted expressions: - -```iex -iex> quote do: sum(1, 2 + 3, 4) -{:sum, [], [1, {:+, [context: Elixir, import: Kernel], [2, 3]}, 4]} -``` - -Sometimes when working with quoted expressions, it may be useful to get the textual code representation back. This can be done with `Macro.to_string/1`: - -```iex -iex> Macro.to_string(quote do: sum(1, 2 + 3, 4)) -"sum(1, 2 + 3, 4)" -``` - -In general, the tuples above are structured according to the following format: - -```elixir -{ tuple | atom, list, list | atom } -``` - -* The first element is an atom or another tuple in the same representation; -* The second element is a keyword list containing metadata, like numbers and contexts; -* The third element is either a list of arguments for the function call or an atom. When this element is an atom, it means the tuple represents a variable. - -Besides the tuple defined above, there are five Elixir literals that, when quoted, return themselves (and not a tuple). They are: - -```elixir -:sum #=> Atoms -1.0 #=> Numbers -[1, 2] #=> Lists -"strings" #=> Strings -{key, value} #=> Tuples with two elements -``` - -Most Elixir code has a straight-forward translation to its underlying quoted expression. We recommend you try out different code samples and see what the results are. For example, what does `String.upcase("foo")` expand to? We have also learned that `if(true, do: :this, else: :that)` is the same as `if true do :this else :that end`. How does this affirmation hold with quoted expressions? - -## 1.2 Unquoting - -Quote is about retrieving the inner representation of some particular chunk of code. However, sometimes it may be necessary to inject some other particular chunk of code inside the representation we want to retrieve. - -For example, imagine you have a variable `number` which contains the number you want to inject inside a quoted expression. The number can be injected into the quoted representation by using `unquote`: - -```iex -iex> number = 13 -iex> Macro.to_string(quote do: 11 + unquote(number)) -"11 + 13" -``` - -`unquote` can even be used to inject function names: - -```iex -iex> fun = :hello -iex> Macro.to_string(quote do: unquote(fun)(:world)) -"hello(:world)" -``` - -In some cases, it may be necessary to inject many values inside a list. For example, imagine you have a list containing `[1, 2, 6]` and we want to inject `[3, 4, 5]` into it. Using `unquote` won't yield the desired result: - -```iex -iex> inner = [3, 4, 5] -iex> Macro.to_string(quote do: [1, 2, unquote(inner), 6]) -"[1, 2, [3, 4, 5], 6]" -``` - -That's when `unquote_splicing` becomes handy: - -```iex -iex> inner = [3, 4, 5] -iex> Macro.to_string(quote do: [1, 2, unquote_splicing(inner), 6]) -"[1, 2, 3, 4, 5, 6]" -``` - -Unquoting is very useful when working with macros. When writing macros, developers are able to receive code chunks and inject them inside other code chunks, which can be used to transform code or write code that generates code during compilation. - -## 1.3 Escaping - -As we saw at the beginning of this chapter, only some values are valid quoted expressions in Elixir. For example, a map is not a valid quoted expression. Neither is a tuple with four elements. However, such values *can* be expressed as a quoted expression: - -```iex -iex> quote do: %{1 => 2} -{:%{}, [], [{1, 2}]} -``` - -In some cases, you may need to inject such *values* into *quoted expressions*. To do that, we need to first escape those values into quoted expressions with the help of `Macro.escape/1`: - -```iex -iex> map = %{hello: :world} -iex> Macro.escape(map) -{:%{}, [], [hello: :world]} -``` - -Macros receive quoted expressions and must return quoted expressions. However, sometimes during the execution of a macro, you may need to work with values and making a distinction between values and quoted expressions will be required. - -In other words, it is important to make a distinction between a regular Elixir value (like a list, a map, a process, a reference, etc) and a quoted expression. Some values, such as integers, atoms and strings, have a quoted expression equal to the value itself. Other values, like maps, need to be explicitly converted. Finally, values like functions and references cannot be converted to a quoted expression at all. - -You can read more about `quote` and `unquote` in the [`Kernel.SpecialForms` module](/docs/stable/elixir/Kernel.SpecialForms.html). Documentation for `Macro.escape/1` and other functions related to quoted expressions can be found in the [`Macro` module](/docs/stable/elixir/Macro.html). - -In this introduction we have laid the groundwork to finally write our first macro, so let's move to the next chapter. diff --git a/getting_started/meta/2.markdown b/getting_started/meta/2.markdown deleted file mode 100644 index 4da0fa569..000000000 --- a/getting_started/meta/2.markdown +++ /dev/null @@ -1,284 +0,0 @@ ---- -layout: getting_started -title: 2 Macros -guide: 2 ---- - -# {{ page.title }} - -
      - -Macros can be defined in Elixir using `defmacro/2`. - -> For this chapter, we will be using files instead of running code samples in IEx. That's because the code samples will span multiple lines of code and typing them all in IEx can be counter-productive. You should be able to run the code samples by saving them into a `macros.exs` file and running it with `elixir macros.exs` or `iex macros.exs`. - -## 2.1 Our first macro - -In order to better understand how macros work, let's create a new module where we are going to implement `unless`, which does the opposite of `if`, as a macro and as a function: - -```elixir -defmodule Unless do - def fun_unless(clause, expression) do - if(!clause, do: expression) - end - - defmacro macro_unless(clause, expression) do - quote do - if(!unquote(clause), do: unquote(expression)) - end - end -end -``` - -The function receives the arguments and passes them to `if`. However, as we learned in the previous chapter, the macro will receive quoted expressions, inject them into the quote, and finally return another quoted expression. - -Let's start `iex` with the module above: - - $ iex macros.exs - -And play with those definitions: - -```iex -iex> require Unless -iex> Unless.macro_unless true, IO.puts "this should never be printed" -nil -iex> Unless.fun_unless true, IO.puts "this should never be printed" -"this should never be printed" -nil -``` - -Note that in our macro implementation, the sentence was not printed, although it was printed in our function implementation. That's because the arguments to a function call are evaluated before calling the function. However, macros do not evaluate their arguments. Instead, they receive the arguments as quoted expressions which are then transformed into other quoted expressions. In this case, we have rewritten our `unless` macro to become an `if` behind the scenes. - -In other words, when invoked as: - -```elixir -Unless.macro_unless true, IO.puts "this should never be printed" -``` - -Our `macro_unless` macro received the following: - -{% raw %} -```elixir -macro_unless(true, {{:., [], [IO, :puts], [], ["this should never be printed"]}}) -``` -{% endraw %} - -And it then returned a quoted expression as follows: - -{% raw %} -```elixir -{:if, [], [ - {:!, [], [true]}, - {{:., [], [IO, :puts], [], ["this should never be printed"]}}]} -``` -{% endraw %} - -We can actually verify that this is the case by using `Macro.expand_once/2`: - -```iex -iex> expr = quote do: Unless.macro_unless(true, IO.puts "this should never be printed") -iex> res = Macro.expand_once(expr, __ENV__) -iex> IO.puts Macro.to_string(res) -if(!true) do - IO.puts("this should never be printed") -end -:ok -``` - -`Macro.expand_once/2` receives a quoted expression and expands it according to the current environment. In this case, it expanded/invoked the `Unless.macro_unless/2` macro and returned its result. We then proceeded to convert the returned quoted expression to a string and print it (we will talk about `__ENV__` later in this chapter). - -That's what macros are all about. They are about receiving quoted expressions and transforming them into something else. In fact, `unless/2` in Elixir is implemented as a macro: - -```elixir -defmacro unless(clause, options) do - quote do - if(!unquote(clause), do: unquote(options)) - end -end -``` - -Constructs such as `unless/2`, `defmacro/2`, `def/2`, `defprotocol/2`, and many others used throughout this getting started guide are implemented in pure Elixir, often as a macros. This means that the constructs being used to build the language can be used by developers to extend the language to the domains they are working on. - -We can define any function and macro we want, including ones that override the built-in definitions provided by Elixir. The only exceptions are Elixir special forms which are not implemented in Elixir and therefore cannot be overridden, [the full list of special forms is available in `Kernel.SpecialForms`](/docs/stable/elixir/Kernel.SpecialForms.html). - -## 2.2 Macros hygiene - -Elixir macros have late resolution. This guarantees that a variable defined inside a quote won't conflict with a variable defined in the context where that macro is expanded. For example: - -```elixir -defmodule Hygiene do - defmacro no_interference do - quote do: a = 1 - end -end - -defmodule HygieneTest do - def go do - require Hygiene - a = 13 - Hygiene.no_interference - a - end -end - -HygieneTest.go -# => 13 -``` - -In the example above, even though the macro injects `a = 1`, it does not affect the variable `a` defined by the `go` function. If a macro wants to explicitly affect the context, it can use `var!`: - -```elixir -defmodule Hygiene do - defmacro interference do - quote do: var!(a) = 1 - end -end - -defmodule HygieneTest do - def go do - require Hygiene - a = 13 - Hygiene.interference - a - end -end - -HygieneTest.go -# => 1 -``` - -Variable hygiene only works because Elixir annotates variables with their context. For example, a variable `x` defined on line 3 of a module would be represented as: - - {:x, [line: 3], nil} - -However, a quoted variable is represented as: - -```elixir -defmodule Sample do - def quoted do - quote do: x - end -end - -Sample.quoted #=> {:x, [line: 3], Sample} -``` - -Notice that the third element in the quoted variable is the atom `Sample`, instead of `nil`, which marks the variable as coming from the `Sample` module. Therefore, Elixir considers these two variables as coming from different contexts and handles them accordingly. - -Elixir provides similar mechanisms for imports and aliases too. This guarantees that a macro will behave as specified by its source module rather than conflicting with the target module where the macro is expanded. Hygiene can be bypassed under specific situations by using macros like `var!/2` and `alias!/2`, although one must be careful when using those as they directly change the user environment. - -Sometimes variable names might be dynamically created. In such cases, `Macro.var/2` can be used to define new variables: - -```elixir -defmodule Sample do - defmacro initialize_to_char_count(variables) do - Enum.map variables, fn(name) -> - var = Macro.var(name, nil) - length = Atom.to_string(name)|> String.length - quote do - unquote(var) = unquote(length) - end - end - end - - def run do - initialize_to_char_count [:red, :green, :yellow] - [red, green, yellow] - end -end - -> Sample.run #=> [3, 5, 6] -``` - -Take note of the second argument to `Macro.var/2`. This is the context being used and will determine hygiene as described in this section. - -## 2.3 The environment - -When calling `Macro.expand_once/2` earlier in this chapter, we used the special form `__ENV__`. - -`__ENV__` returns an instance of `Macro.Env` which contains useful information about the compilation environment, including the current module, file and line, all variables defined in the current scope, as well as imports, requires and so on: - -```iex -iex> __ENV__.module -nil -iex> __ENV__.file -"iex" -iex> __ENV__.requires -[IEx.Helpers, Kernel, Kernel.Typespec] -iex> require Integer -nil -iex> __ENV__.requires -[IEx.Helpers, Integer, Kernel, Kernel.Typespec] -``` - -Many of the functions in the `Macro` module expect an environment. You can read more about them in [the docs for the `Macro` module](/docs/stable/elixir/Macro.html) and learn more about the compilation environment with [`Macro.Env`](/docs/stable/elixir/Macro.Env.html). - -## 2.4 Private macros - -Elixir also supports private macros via `defmacrop`. As private functions, these macros are only available inside the module that defines them, and only at compilation time. - -It is important that a macro is defined before its usage. Failing to define a macro before its invocation will raise an error at runtime, since the macro won't be expanded and will be translated to a function call: - -```elixir -iex> defmodule Sample do -...> def four, do: two + two -...> defmacrop two, do: 2 -...> end -** (CompileError) iex:2: function two/0 undefined -``` - -## 2.5 Write macros responsibly - -Macros are a powerful construct and Elixir provides many mechanisms to ensure they are used responsibly: - -* Macros are hygienic: by default, variables defined inside the macro are not going to affect the user code. Furthermore, function calls and aliases available in the macro context are not going to leak into the user context; - -* Macros are lexical: it is impossible to inject code or macros globally. Before using a macro, you need to explicitly `require` or `import` the module that defines the macro; - -* Macros are explicit: it is impossible to run a macro without explicitly invoking it. For example, some languages allow developers to completely rewrite functions behind the scenes, often via parse transforms or via some reflection mechanisms. In Elixir, a macro must be explicitly invoked in the caller; - -* Macros' language is clear: many languages provide syntax shortcuts for `quote` and `unquote`. In Elixir, we preferred to have them explicitly spelled out, in order to clearly delimit the boundaries of a macro definition and its quoted expressions; - -Even if Elixir attempts its best to provide a safe environment, the major responsibility still falls on the developers. That's why the first rule of the macro club is **write macros responsibly**. Macros are harder to write than ordinary Elixir functions and it's considered to be bad style to use them when they're not necessary. Elixir already provides elegant mechanisms to write your every day code and macros should be saved as a last resort. - -And, if you ever need to resort to macros, remember that macros are not your API. Keep your macro definitions short, including its quoted contents. For example, instead of writing a macro like this: - -```elixir -defmodule MyModule do - defmacro my_macro(a, b, c) do - quote do - do_this(unquote(a)) - ... - do_that(unquote(b)) - ... - and_that(unquote(c)) - end - end -end -``` - -write: - -```elixir -defmodule MyModule do - defmacro my_macro(a, b, c) do - quote do - # Keep what you need to do here to a minimum - # and move everything else to a function - do_this_that_and_that(unquote(a), unquote(b), unquote(c)) - end - end - - def do_this_that_and_that(a, b, c) do - do_this(a) - ... - do_that(b) - ... - and_that(c) - end -end -``` - -This makes your code clearer and easier to test and maintain, as you can invoke `do_this_that_and_that/3` directly. It also helps you design an actual API for developers that does not rely on macros. - -With those lessons, we finish our introduction to macros. The next chapter is a brief discussion on DSLs, showing how we can mix macros and module attributes to annotate and extend modules and functions. diff --git a/getting_started/meta/3.markdown b/getting_started/meta/3.markdown deleted file mode 100644 index 005237eb4..000000000 --- a/getting_started/meta/3.markdown +++ /dev/null @@ -1,166 +0,0 @@ ---- -layout: getting_started -title: 3 Domain Specific Languages -guide: 3 -last: true ---- - -# {{ page.title }} - -
      - -[Domain Specific Languages](https://en.wikipedia.org/wiki/Domain-specific_language) allow developers to tailor their application to a particular domain. There are many language features that, when used in combination, can aid developers to write Domain Specific Languages. In this chapter we will focus on how macros and module attributes can be used together to create domain specific modules that are focused on solving one particular problem. As an example, we will write a very simple module to define and run tests. - -The goal is to build a module named `TestCase` that allows us to write the following: - -```elixir -defmodule MyTest do - use TestCase - - test "arithmetic operations" do - 4 = 2 + 2 - end - - test "list operations" do - [1, 2, 3] = [1, 2] ++ [3] - end -end - -MyTest.run -``` - -In the example above, by using `TestCase`, we can write tests using the `test` macro, which defines a function named `run` to automatically run all tests for us. Our prototype will simply rely on the match operator (`=`) as a mechanism to do assertions. - -## 3.1 The `test` macro - -Let's start by creating a module that simply defines and imports the `test` macro when used: - -```elixir -defmodule TestCase do - # Callback invoked by `use`. - # - # For now it simply returns a quoted expression that - # imports the module itself into the user code. - @doc false - defmacro __using__(_opts) do - quote do - import TestCase - end - end - - @doc """ - Defines a test case with the given description. - - ## Examples - - test "arithmetic operations" do - 4 = 2 + 2 - end - - """ - defmacro test(description, do: block) do - function_name = String.to_atom("test " <> description) - quote do - def unquote(function_name)(), do: unquote(block) - end - end -end -``` - -Assuming we defined `TestCase` in a file named `tests.exs`, we can open it up by running `iex tests.exs` and define our first tests: - -```iex -iex> defmodule MyTest do -...> use TestCase -...> -...> test "hello" do -...> "hello" = "world" -...> end -...> end -``` - -For now we don't have a mechanism to run tests, but we know that a function named "test hello" was defined behind the scenes. When we invoke it, it should fail: - -```iex -iex> MyTest."test hello"() -** (MatchError) no match of right hand side value: "world" -``` - -## 3.2 Storing information with attributes - -In order to finish our `TestCase` implementation, we need to be able to access all defined test cases. One way of doing this is by retrieving the tests at runtime via `__MODULE__.__info__(:functions)`, which returns a list of all functions in a given module. However, considering that we may want to store more information about each test besides the test name, a more flexible approach is required. - -When discussing module attributes in earlier chapters, we mentioned how they can be used as temporary storage. That's exactly the property we will apply in this section. - -In the `__using__/1` implementation, we will initialize a module attribute named `@tests` to an empty list, then store the name of each defined test in this attribute so the tests can be invoked from the `run` function. - -Here is the updated code for the `TestCase` module: - -```elixir -defmodule TestCase do - @doc false - defmacro __using__(_opts) do - quote do - import TestCase - - # Initialize @tests to an empty list - @tests [] - - # Invoke TestCase.__before_compile__/1 before the module is compiled - @before_compile TestCase - end - end - - @doc """ - Defines a test case with the given description. - - ## Examples - - test "arithmetic operations" do - 4 = 2 + 2 - end - - """ - defmacro test(description, do: block) do - function_name = String.to_atom("test " <> description) - quote do - # Prepend the newly defined test to the list of tests - @tests [unquote(function_name)|@tests] - def unquote(function_name)(), do: unquote(block) - end - end - - # This will be invoked right before the target module is compiled - # giving us the perfect opportunity to inject the `run/0` function - @doc false - defmacro __before_compile__(env) do - quote do - def run do - Enum.each @tests, fn name -> - IO.puts "Running #{name}" - apply(__MODULE__, name, []) - end - end - end - end -end -``` - -By starting a new IEx session, we can now define our tests and run them: - -```iex -iex> defmodule MyTest do -...> use TestCase -...> -...> test "hello" do -...> "hello" = "world" -...> end -...> end -iex> MyTest.run -Running test hello -** (MatchError) no match of right hand side value: "world" -``` - -Although we have overlooked some details, this is the main idea behind creating domain specific modules in Elixir. Macros enable us to return quoted expressions that are executed in the caller, which we can then use to transform code and store relevant information in the target module via module attributes. Finally, callbacks such as `@before_compile` allow us to inject code into the module when its definition is complete. - -Besides `@before_compile`, there are other useful module attributes like `@on_definition` and `@after_compile`, which you can read more about in [the docs for the `Module` module](/docs/stable/elixir/Module.html). You can also find useful information about macros and the compilation environment in the documentation for the [`Macro` module](/docs/stable/elixir/Macro.html) and [`Macro.Env`](/docs/stable/elixir/Macro.Env.html). diff --git a/getting_started/mix_otp/1.markdown b/getting_started/mix_otp/1.markdown deleted file mode 100644 index a7d93cacc..000000000 --- a/getting_started/mix_otp/1.markdown +++ /dev/null @@ -1,236 +0,0 @@ ---- -layout: getting_started -title: 1 Introduction to Mix -guide: 1 ---- - -# {{ page.title }} - -
      - -In this guide, we will learn how to build a complete Elixir application, with its own supervision tree, configuration, tests and more. - -The application works as a distributed key-value store. We are going to organize key-value pairs into buckets and distribute those buckets across multiple nodes. We will also build a simple client that allows us to connect to any of those nodes and send requests such as: - -``` -CREATE shopping -OK - -PUT shopping milk 1 -OK - -PUT shopping eggs 3 -OK - -GET shopping milk -1 -OK - -DELETE shopping eggs -OK -``` - -In order to build our key-value application, we are going to use three main tools: - -* OTP is a set of libraries that ships with Erlang. Erlang developers use OTP to build robust, fault-tolerant applications. In this chapter we will explore how many aspects from OTP integrate with Elixir, including supervision trees, event managers and more; - -* Mix is a build tool that ships with Elixir that provides tasks for creating, compiling, testing your application, managing its dependencies and much more; - -* ExUnit is a test-unit based framework that ships with Elixir; - -In this chapter, we will create our first project using Mix and explore different features in OTP, Mix and ExUnit as we go. - -> Note: this guide requires Elixir v0.15.0 or later. You can check your Elixir version with `elixir -v` and install a more recent version if required by following the steps described in [the first chapter of the Getting Started guide](/getting_started/1.html). -> -> If you have any questions or improvements to the guide, please let us know in [our mailing list](https://groups.google.com/d/forum/elixir-lang-talk) or [issues tracker](http://github.com/elixir-lang/elixir-lang.github.com/issues) respectively. Your input is really important to help us guarantee the guides are accessible and up to date! - -## 1.1 Our first project - -When you install Elixir, besides getting the `elixir`, `elixirc` and `iex` executables, you also get an executable Elixir script named `mix`. - -Let's create our first project by invoking `mix new` from the command line. We'll pass the project name as argument (`kv`, in this case), and tell mix that our main module should be the all-uppercase `KV`, instead of the default, which would have been `Kv`: - - $ mix new kv --module KV - -Mix will create a directory named `kv` with a few files in it: - - * creating README.md - * creating .gitignore - * creating mix.exs - * creating config - * creating config/config.exs - * creating lib - * creating lib/kv.ex - * creating test - * creating test/test_helper.exs - * creating test/kv_test.exs - -Let's take a brief look at those generated files. - -> Note: Mix is an Elixir executable. This means that in order to run `mix`, you need to have elixir's executable in your PATH. If not, you can run it by passing the script as argument to elixir: -> -> $ bin/elixir bin/mix new kv --module KV -> -> Note that you can also execute any script in your PATH from Elixir via the -S option: -> -> $ bin/elixir -S mix new kv --module KV -> -> When using -S, elixir finds the script wherever it is in your PATH and executes it. - -## 1.2 Project compilation - -A file named `mix.exs` was generated inside our new project and its main responsibility is to configure our project. Let's take a look at it (comments removed): - -```elixir -defmodule KV.Mixfile do - use Mix.Project - - def project do - [app: :kv, - version: "0.0.1", - deps: deps] - end - - def application do - [applications: [:logger]] - end - - defp deps do - [] - end -end -``` - -Our `mix.exs` defines two public functions: `project`, which returns project configuration like the project name and version, and `application`, which is used to generate an application file. - -There is also a private function named `deps`, which is invoked from the `project` function, that defines our project dependencies. Defining `deps` as a separate function is not required, but it helps keep the project configuration tidy. - -Mix also generates a file at `lib/kv.ex` with a simple module definition: - -```elixir -defmodule KV do -end -``` - -This structure is enough to compile our project: - - $ mix compile - -Will generate: - - Compiled lib/kv.ex - Generated kv.app - -Notice the `lib/kv.ex` file was compiled and `kv.app` file was generated. This `.app` file is generated with the information from the `application/0` function in the `mix.exs` file. We will further explore `mix.exs` configuration features in future chapters. - -Once the project is compiled, you can start an `iex` session inside the project by running: - - $ iex -S mix - -## 1.3 Running tests - -Mix also generated the appropriate structure for running our project tests. Mix projects usually follow the convention of having a `_test.exs` file in the `test` directory for each file in the `lib` directory. For this reason, we can already find a `test/kv_test.exs` corresponding to our `lib/kv.ex` file. It doesn't do much at this point: - -```elixir -defmodule KVTest do - use ExUnit.Case - - test "the truth" do - assert 1 + 1 == 2 - end -end -``` - -It is important to note a couple things: - -1. the test file is an Elixir script file (`.exs`). This is convenient because we don't need to compile test files before running them; - -2. we define a test module named `KVTest`, use [`ExUnit.Case`](/docs/stable/ex_unit/ExUnit.Case.html) to inject the testing API and define a simple test using the `test/2` macro; - -Mix also generated a file named `test/test_helper.exs` which is responsible for setting up the test framework: - -```elixir -ExUnit.start -``` - -This file will be automatically required by Mix every time before we run our tests. We can run tests with `mix test`: - - Compiled lib/kv.ex - Generated kv.app - . - - Finished in 0.04 seconds (0.04s on load, 0.00s on tests) - 1 tests, 0 failures - - Randomized with seed 540224 - -Notice that by running `mix test`, Mix has compiled the source files and generated the application file once again. This happens because Mix supports multiple environments, which we will explore in the next section. - -Furthermore, you can see that ExUnit prints a dot for each successful test and automatically randomizes tests too. Let's make the test fail on purpose and see what happens. - -Change the assertion in `test/kv_test.exs` to the following: - -```elixir -assert 1 + 1 == 3 -``` - -Now run `mix test` again (notice this time there was no compilation): - - 1) test the truth (KVTest) - test/kv_test.exs:4 - Assertion with == failed - code: 1 + 1 == 3 - lhs: 2 - rhs: 3 - stacktrace: - test/kv_test.exs:5 - - Finished in 0.05 seconds (0.05s on load, 0.00s on tests) - 1 tests, 1 failures - -For each failure, ExUnit prints a detailed report, containing the test name with the test case, the code that failed and the values for the left-hand side (lhs) and right-hand side (rhs) of the `==` operator. - -In the second line of the failure, right below the test name, there is the location the test was defined. If you copy the test location in the second line, containing the file and line, and paste it in-front of `mix test`, Mix will load and run just that particular test: - - $ mix test test/kv_test.exs:4 - -This shortcut will be extremely useful as we build our project, allowing us to quickly iterate by running just a specific test. - -Finally, the stacktrace relates to the failure itself, giving information about the test and often the place the failure was generated from within the source files. - -## 1.4 Environments - -Mix supports the concept of "environments". They allow a developer to customize compilation and other options for specific scenarios. By default, Mix understands three environments: - -* `:dev` - the one in which mix tasks (like `compile`) run by default -* `:test` - used by `mix test` -* `:prod` - the one you will use to put your project in production - -> Note: If you add dependencies to your project, they will not inherit your project's environment, but instead run with their `:prod` environment settings! - -By default, these environments behave the same and all configuration we have seen so far will affect all three environments. Customization per environment can be done by accessing [the `Mix.env` function](/docs/stable/mix/Mix.html#env/1) in your `mix.exs` file, which returns the current environment as an atom: - -```elixir -def project do - [deps_path: deps_path(Mix.env)] -end - -defp deps_path(:prod), do: "prod_deps" -defp deps_path(_), do: "deps" -``` - -Mix will default to the `:dev` environment, except for the `test` task that will default to the `:test` environment. The environment can be changed via the `MIX_ENV` environment variable: - - $ MIX_ENV=prod mix compile - -## 1.5 Exploring - -There is much more to Mix, and we will continue to explore it as we build our project. A [general overview is available on the Mix documentation](http://elixir-lang.org/docs/stable/mix). - -Keep in mind that you can always invoke the help task to list all available tasks: - - $ mix help - -You can get further information about a particular task by invoking `mix help TASK`. - -Let's write some code! diff --git a/getting_started/mix_otp/10.markdown b/getting_started/mix_otp/10.markdown deleted file mode 100644 index 3084f0471..000000000 --- a/getting_started/mix_otp/10.markdown +++ /dev/null @@ -1,350 +0,0 @@ ---- -layout: getting_started -title: 10 Distributed tasks and configuration -guide: 10 -last: true ---- - -# {{ page.title }} - -
      - -In this last chapter, we will go back to the `:kv` application and add a routing layer that allows us to distribute requests between nodes based on the bucket name. - -The routing layer will receive a routing table of the following format: - - [{?a..?m, :"foo@computer-name"}, - {?n..?z, :"bar@computer-name"}] - -The router will check the first byte of the bucket name against the table and dispatch to the appropriate node based on that. For example, a bucket starting with the letter "a" (`?a` represents the Unicode codepoint of the letter "a") will be dispatched to node `foo@computer-name`. - -If the matching entry points to the node evaluating the request, then we've finished routing, and this node will perform the requested operation. If the matching entry points to a different node, we'll pass the request to this node, which will look at its own routing table (which may be different from the one in the first node) and act accordingly. If no entry matches, an error will be raised. - -You may wonder why we don't simply tell the node we find in our routing table to perform the requested operation directly, but instead pass the routing request on to that node to process. While a routing table as simple as the one above might reasonably be shared between all nodes, passing on the routing request in this way makes it much simpler to break the routing table into smaller pieces as our application grows. Perhaps at some point, `foo@computer-name` will only be responsible for routing bucket requests, and the buckets it handles will be dispatched to different nodes. In this way, `bar@computer-name` does not need to know anything about this change. - -> Note: we will be using two nodes in the same machine throughout this chapter. You are free to use two (or more) different machines in the same network but you need to do some prep work. First of all, you need to ensure all machines have a `~/.erlang.cookie` file with exactly the same value. Second, you need to guarantee [epmd](http://www.erlang.org/doc/man/epmd.html) is running on a port that is not blocked (you can run `epmd -d` for debug info). Third, if you want to learn more about distribution in general, we recommend [this great Distribunomicon chapter from Learn You Some Erlang](http://learnyousomeerlang.com/distribunomicon). - -## 10.1 Our first distributed code - -Elixir ships with facilities to connect nodes and exchange information between them. In fact, we use the same concepts of processes, message passing and receiving messages when working in a distributed environment because Elixir processes are *location transparent*. This means that when sending a message, it doesn't matter if the recipient process is on the same node or on another node, the VM will be able to deliver the message in both cases. - -In order to run distributed code, we need to start the VM with a name. The name can be short (when in the same network) or long (requires the full computer address). Let's start a new IEx session: - - $ iex --sname foo - -You can see now the prompt is slightly different and shows the node name followed by the computer name: - - Interactive Elixir - press Ctrl+C to exit (type h() ENTER for help) - iex(foo@jv)1> - -My computer is named `jv`, so I see `foo@jv` in the example above, but you will get a different result. We will use `jv@computer-name` in the following examples and you should update them accordingly when trying out the code. - -Let's define a module named `Hello` in this shell: - -```iex -iex> defmodule Hello do -...> def world, do: IO.puts "hello world" -...> end -``` - -If you have another computer on the same network with both Erlang and Elixir installed, you can start another shell on it. If you don't, you can simply start another IEx session in another terminal. In either case, give it the short name of `bar`: - - $ iex --sname bar - -Note that inside this new IEx session, we cannot access `Hello.world/0`: - -```iex -iex> Hello.world -** (UndefinedFunctionError) undefined function: Hello.world/0 - Hello.world() -``` - -However we can spawn a new process on `foo@computer-name` from `bar@computer-name`! Let's give it a try (where `@computer-name` is the one you see locally): - -```iex -iex> Node.spawn_link :"foo@computer-name", fn -> Hello.world end -#PID<9014.59.0> -hello world -``` - -Elixir spawned a process on another node and returned its pid. The code then executed on the other node where the `Hello.world/0` function exists and invoked that function. Note that the result of "hello world" was printed on the current node `bar` and not on `foo`. In other words, the message to be printed was sent back from `foo` to `bar`. This happens because the process spawned on the other node (`foo`) still has the group leader of the current node (`bar`). We have briefly talked about group leaders in the [IO chapter](/getting_started/12.html). - -We can send and receive message from the pid returned by `Node.spawn_link/2` as usual. Let's try a quick ping-pong example: - -```iex -iex> pid = Node.spawn_link :"foo@computer-name", fn -> -...> receive do -...> {:ping, client} -> send client, :pong -...> end -...> end -#PID<9014.59.0> -iex> send pid, {:ping, self} -{:ping, #PID<0.73.0>} -iex> flush -:pong -:ok -``` - -From our quick exploration, we could conclude that we should simply use `Node.spawn_link/2` to spawn processes on a remote node every time we need to do a distributed computation. However we have learned throughout this guide that spawning processes outside of supervision trees should be avoided if possible, so we need to look for other options. - -There are three better alternatives to `Node.spawn_link/2` that we could use in our implementation: - -1. We could use Erlang's [:rpc](http://erlang.org/doc/man/rpc.html) module to execute functions on a remote node. Inside the `bar@computer-name` shell above, you can call `:rpc.call(:"foo@computer-name", Hello, :world, [])` and it will print "hello world" - -2. We could have a server running on the other node and send requests to that node via the [GenServer](/docs/stable/elixir/GenServer.html) API. For example, you can call a remote named server using `GenServer.call({name, node}, arg)` or simply passing the remote process PID as first argument - -3. We could use tasks, which we have learned about in the previous chapter, as they can be spawned on both local and remote nodes - -The options above have different properties. Both `:rpc` and using a GenServer would serialize your requests on a single server, while tasks are effectively running asynchronously on the remote node, with the only serialization point being the spawning done by the supervisor. - -For our routing layer, we are going to use tasks, but feel free to explore the other alternatives too. - -## 10.2 async/await - -So far we have explored tasks that are started and run in isolation, with no regard for their return value. However, sometimes it is useful to run a task to compute a value and read its result later on. For this, tasks also provide the `async/await` pattern: - -```elixir -task = Task.async(fn -> compute_something_expensive end) -res = compute_something_else() -res + Task.await(task) -``` - -`async/await` provides a very simple mechanism to compute values concurrently. Not only that, `async/await` can also be used with the same [`Task.Supervisor`](/docs/stable/elixir/Task.Supervisor.html) we have used in previous chapters. We just need to call `Task.Supervisor.async/2` instead of `Task.Supervisor.start_child/2` and use `Task.await/2` to read the result later on. - -## 10.3 Distributed tasks - -Distributed tasks are exactly the same as supervised tasks. The only difference is that we pass the node name when spawning the task on the supervisor. Open up `lib/kv/supervisor.ex` from the `:kv` application. Let's add a task supervisor to the tree: - -```elixir -supervisor(Task.Supervisor, [[name: KV.RouterTasks]]), -``` - -Now, let's start two named nodes again, but inside the `:kv` application: - - $ iex --sname foo -S mix - $ iex --sname bar -S mix - -From inside `bar@computer-name`, we can now spawn a task directly on the other node via the supervisor: - -```iex -iex> task = Task.Supervisor.async {KV.RouterTasks, :"foo@computer-name"}, fn -> -...> {:ok, node()} -...> end -%Task{pid: #PID<12467.88.0>, ref: #Reference<0.0.0.400>} -iex> Task.await(task) -{:ok, :"foo@computer-name"} -``` - -Our first distributed task is straightforward: it simply gets the name of the node the task is running on. With this knowledge in hand, let's finally write the routing code. - -## 10.4 Routing layer - -Create a file at `lib/kv/router.ex` with the following contents: - -```elixir -defmodule KV.Router do - @doc """ - Dispatch the given `mod`, `fun`, `args` request - to the appropriate node based on the `bucket`. - """ - def route(bucket, mod, fun, args) do - # Get the first byte of the binary - first = :binary.first(bucket) - - # Try to find an entry in the table or raise - entry = - Enum.find(table, fn {enum, node} -> - first in enum - end) || no_entry_error(bucket) - - # If the entry node is the current node - if elem(entry, 1) == node() do - apply(mod, fun, args) - else - sup = {KV.RouterTasks, elem(entry, 1)} - Task.Supervisor.async(sup, fn -> - KV.Router.route(bucket, mod, fun, args) - end) |> Task.await() - end - end - - defp no_entry_error(bucket) do - raise "could not find entry for #{inspect bucket} in table #{inspect table}" - end - - @doc """ - The routing table. - """ - def table do - # Replace computer-name with your local machine name. - [{?a..?m, :"foo@computer-name"}, - {?n..?z, :"bar@computer-name"}] - end -end -``` - -Let's write a test to verify our router works. Create a file named `test/kv/router_test.exs` containing: - -```elixir -defmodule KV.RouterTest do - use ExUnit.Case, async: true - - test "route requests accross nodes" do - assert KV.Router.route("hello", Kernel, :node, []) == - :"foo@computer-name" - assert KV.Router.route("world", Kernel, :node, []) == - :"bar@computer-name" - end - - test "raises on unknown entries" do - assert_raise RuntimeError, ~r/could not find entry/, fn -> - KV.Router.route(<<0>>, Kernel, :node, []) - end - end -end -``` - -The first test simply invokes `Kernel.node/0`, which returns the name of the current node, based on the bucket names "hello" and "world". According to our routing table so far, we should get `foo@computer-name` and `bar@computer-name` as responses, respectively. - -The second test just checks that the code raises for unknown entries. - -In order to run the first test, we need to have two nodes running. Let's restart the node named `bar`, which is going to be used by tests: - - $ iex --sname bar -S mix - -And now run tests with: - - $ elixir --sname foo -S mix test - -Our test should successfuly pass. Excellent! - -## 10.5 Test filters and tags - -Although our tests pass, our testing structure is getting more complex. In particular, running tests with only `mix test` causes failures in our suite, since our test requires a connection to another node. - -Luckily, ExUnit ships with a facility to tag tests, allowing us to run specific callbacks or even filter tests altogether based on those tags. - -All we need to do to tag a test is simply call `@tag` before the test name. Back to `test/kv/routest_test.exs`, let's add a `:distributed` tag: - -```elixir -@tag :distributed -test "route requests accross nodes" do -``` - -Writing `@tag :distributed` is equivalent to writing `@tag distributed: true`. - -With the test properly tagged, we can now check if the node is alive on the network and, if not, we can exclude all distributed tests. Open up `test/test_helper.exs` inside the `:kv` application and add the following: - -```elixir -exclude = - if Node.alive?, do: [], else: [distributed: true] - -ExUnit.start(exclude: exclude) -``` - -Now run tests with `mix test`: - - $ mix test - Excluding tags: [distributed: true] - - ....... - - Finished in 0.1 seconds (0.1s on load, 0.01s on tests) - 7 tests, 0 failures - -This time all tests passed and ExUnit warned us that distributed tests were being excluded. If you run tests with `$ elixir --sname foo -S mix test`, one extra test should run and successfully pass as long as the `bar@computer-name` node is available. - -The `mix test` command also allows us to dynamically include and exclude tags. For example, we can run `$ mix test --include distributed` to run distributed tests regardless of the value set in `test/test_helper.exs`. We could also pass `--exclude` to exclude a particular tag from the command line. Finally, `--only` can be used to run only tests with a particular tag: - - $ elixir --sname foo -S mix test --only distributed - -You can read more about filters, tags and the default tags in [`ExUnit.Case` module documentation](/docs/stable/ex_unit/ExUnit.Case.html). - -## 10.6 Application environment and configuration - -So far we have hardcoded the routing table into the `KV.Router` module. However, we would like to make the table dynamic. This allows us not only to configure development/test/production, but also to allow different nodes to run with different entries in the routing table. There is a feature of OTP that does exactly that: the application environment. - -Each application has an environment that stores the application specific configuration by key. For example, we could store the routing table in the `:kv` application environment, giving it a default value and allowing other applications to change the table as needed. - -Open up `apps/kv/mix.exs` and change the `application/0` function to return the following: - -```elixir -def application do - [applications: [], - env: [routing_table: []], - mod: {KV, []}] -end -``` - -We have added a new `:env` key to the application. It returns the application default environment, which has an entry of key `:routing_table` and value of an empty list. It makes sense for the application environment to ship with an empty table, as the specific routing table depends on the testing/deployment structure. - -In order to use the application environment in our code, we just need to replace `KV.Router.table/0` with the definition below: - -```elixir -@doc """ -The routing table. -""" -def table do - Application.get_env(:kv, :routing_table) -end -``` - -We use `Application.get_env/2` to read the entry for `:routing_table` in `:kv`'s environment. You can find more information and other functions to manipulate the app environment in the [Application module](/docs/stable/elixir/Application.html). - -Since our routing table is now empty, our distributed test should fail. Restart the apps and re-run tests to see the failure: - - $ iex --sname bar -S mix - $ elixir --sname foo -S mix test --only distributed - -The interesting thing about the application environment is that it can be configured not only for the current application, but for all applications. Such configuration is done by the `config/config.exs` file. For example, we can configure IEx default prompt to another value. Just open `apps/kv/config/config.exs` and add the following to the end: - -```elixir -config :iex, default_prompt: ">>>" -``` - -Start IEx with `iex -S mix` and you can see that the IEx prompt has changed. - -This means we can configure our `:routing_table` directly in the `config/config.exs` file as well: - -```elixir -# Replace computer-name with your local machine nodes. -config :kv, :routing_table, - [{?a..?m, :"foo@computer-name"}, - {?n..?z, :"bar@computer-name"}] -``` - -Restart the nodes and run distributed tests again. Now they should all pass. - -Each application has its own `config/config.exs` file and they are not shared in any way. Configuration can also be set per environment. Read the contents of the config file for the `:kv` application for more information on how to do so. - -Since config files are not shared, if you run tests from the umbrella root, they will fail because the configuration we just added to `:kv` is not available there. However, if you open up `config/config.exs` in the umbrella, it has instructions on how to import config files from children applications. You just need to invoke: - -```elixir -import_config "../apps/kv/config/config.exs" -``` - -The `mix run` command also accepts a `--config` flag, which allows configuration files to be given on demand. This could be used to start different nodes, each with its own specific configuration (for example, different routing tables). - -Overall, the built-in ability to configure applications and the fact that we have built our software as an umbrella application gives us plenty of options when deploying the software. We can: - -* deploy the umbrella application to a node that will work as both TCP server and key-value storage - -* deploy the `:kv_server` application to work only as a TCP server as long as the routing table points only to other nodes - -* deploy only the `:kv` application when we want a node to work only as storage (no TCP access) - -As we add more applications in the future, we can continue controlling our deploy with the same level of granularity, cherry-picking which applications with which configuration are going to production. We can also consider building multiple releases with a tool like [exrm](https://github.com/bitwalker/exrm), which will package the chosen applications and configuration, including the current Erlang and Elixir installations, so we can deploy the application even if the runtime is not pre-installed on the target system. - -Finally, we have learned some new things in this chapter, and they could be applied to the `:kv_server` application as well. We are going to leave the next steps as an exercise: - -* change the `:kv_server` application to read the port from its application environment instead of using the hardcoded value of 4040 - -* change and configure the `:kv_server` application to use the routing functionality instead of dispatching directly to the local `KV.Registry`. For `:kv_server` tests, you can make the routing table simply point to the current node itself - -## 10.7 Summing up - -In this chapter we have built a simple router as a way to explore the distributed features of Elixir and the Erlang VM, and learned how to configure its routing table. This is the last chapter in our Mix and OTP guide. - -Throughout the guide, we have built a very simple distributed key-value store as an opportunity to explore many constructs like generic servers, event managers, supervisors, tasks, agents, applications and more. Not only that, we have written tests for the whole application, getting familiar with ExUnit, and learned how to use the Mix build tool to accomplish a wide range of tasks. - -If you are looking for a distributed key-value store to use in production, you should definitely look into [Riak](http://basho.com/riak/), which also runs in the Erlang VM. In Riak, the buckets are replicated, to avoid data loss, and instead of a router, they use [consistent hashing](http://en.wikipedia.org/wiki/Consistent_hashing) to map a bucket to a node. A consistent hashing algorithm helps reduce the amount of data that needs to be migrated when new nodes to store buckets are added to your infrastructure. diff --git a/getting_started/mix_otp/2.markdown b/getting_started/mix_otp/2.markdown deleted file mode 100644 index 013739121..000000000 --- a/getting_started/mix_otp/2.markdown +++ /dev/null @@ -1,184 +0,0 @@ ---- -layout: getting_started -title: 2 Agent -guide: 2 ---- - -# {{ page.title }} - -
      - -In this chapter, we will create a module named `KV.Bucket`. This module will be responsible for storing our key-value entries in a way that allows reading and modification by different processes. - -If you have skipped the Getting Started guide or if you have read it long ago, be sure to re-read the chapter about [Processes](/getting_started/11.html). We will use it as starting point. - -## 2.1 The trouble with state - -Elixir is an immutable language where nothing is shared by default. If we want to create buckets, store and access them from multiple places, we have two main options in Elixir: - -* Processes -* [ETS (Erlang Term Storage)](http://www.erlang.org/doc/man/ets.html) - -We have talked about processes, while ETS is something new that we will explore later in this guide. When it comes to processes though, we rarely hand-roll our own process, instead we use the abstractions available in Elixir and OTP: - -* [Agent](/docs/stable/elixir/Agent.html) - Simple wrappers around state -* [GenServer](/docs/stable/elixir/GenServer.html) - "Generic servers" (processes) that encapsulate state, provide sync and async calls, support code reloading, and more -* [GenEvent](/docs/stable/elixir/GenEvent.html) - "Generic event" managers that allow publishing events to multiple handlers -* [Task](/docs/stable/elixir/Task.html) - Asynchronous units of computation that allow spawning a process and easily retrieving its result at a later time - -We will explore all of these abstractions in this guide. Keep in mind that they are all implemented on top of processes using the basic features provided by the VM, like `send`, `receive`, `spawn` and `link`. - -## 2.2 Agents - -[Agents](/docs/stable/elixir/Agent.html) are simple wrappers around state. If all you want from a process is to keep state, agents are a great fit. Let's start an `iex` session inside the project with: - - $ iex -S mix - -And play a bit with agents: - -```iex -iex> {:ok, agent} = Agent.start_link fn -> [] end -{:ok, #PID<0.57.0>} -iex> Agent.update(agent, fn list -> ["eggs"|list] end) -:ok -iex> Agent.get(agent, fn list -> list end) -["eggs"] -iex> Agent.stop(agent) -:ok -``` - -We started an agent with an initial state of an empty list. Next, we issue a command to update the state, adding our new item to the head of the list. Finally, we retrieved the whole list. Once we are done with the agent, we can call `Agent.stop/1` to terminate the agent process. - -Let's implement our `KV.Bucket` using agents. But before starting the implementation, let's first write some tests. Create a file at `test/kv/bucket_test.exs` (remember the `.exs` extension) with the following: - -```elixir -defmodule KV.BucketTest do - use ExUnit.Case, async: true - - test "stores values by key" do - {:ok, bucket} = KV.Bucket.start_link - assert KV.Bucket.get(bucket, "milk") == nil - - KV.Bucket.put(bucket, "milk", 3) - assert KV.Bucket.get(bucket, "milk") == 3 - end -end -``` - -Our first test is straightforward. We start a new `KV.Bucket` and perform some `get/2` and `put/3` operations on it, asserting the result. We don't need to explicitly stop the agent because it is linked to the test process and the agent is shut down automatically once the test finishes. - -Also note that we passed the `async: true` option to `ExUnit.Case`. This option makes this test case run in parallel with other test cases that set up the `:async` option. This is extremely useful to speed up our test suite by using multiple cores in our machine. Note though the `:async` option must only be set if the test case does not rely or change any global value. For example, if the test requires writing to the filesystem, registering processes, accessing a database, you must not make it async to avoid race conditions in between tests. - -Regardless of being async or not, our new test should obviously fail, as none of the functionality is implemented. - -In order to fix the failing test, let's create a file at `lib/kv/bucket.ex` with the contents below. Feel free to give a try at implementing the `KV.Bucket` module yourself using agents before peeking the implementation below. - -```elixir -defmodule KV.Bucket do - @doc """ - Starts a new bucket. - """ - def start_link do - Agent.start_link(fn -> HashDict.new end) - end - - @doc """ - Gets a value from the `bucket` by `key`. - """ - def get(bucket, key) do - Agent.get(bucket, &HashDict.get(&1, key)) - end - - @doc """ - Puts the `value` for the given `key` in the `bucket`. - """ - def put(bucket, key, value) do - Agent.update(bucket, &HashDict.put(&1, key, value)) - end -end -``` - -With the `KV.Bucket` module defined, our test should pass! Note that we are using a HashDict to store our state instead of a `Map`, because in the current version of Elixir maps are less efficient when holding a large number of keys. - -## 2.3 ExUnit callbacks - -Before moving on and adding more features to `KV.Bucket`, let's talk about ExUnit callbacks. As you may expect, all `KV.Bucket` tests will require a bucket to be started during setup and stopped after the test. Luckily, ExUnit supports callbacks that allow us to skip such repetitive tasks. - -Let's rewrite the test case to use callbacks: - -```elixir -defmodule KV.BucketTest do - use ExUnit.Case, async: true - - setup do - {:ok, bucket} = KV.Bucket.start_link - {:ok, bucket: bucket} - end - - test "stores values by key", %{bucket: bucket} do - assert KV.Bucket.get(bucket, "milk") == nil - - KV.Bucket.put(bucket, "milk", 3) - assert KV.Bucket.get(bucket, "milk") == 3 - end -end -``` - -We have first defined a setup callback with the help of the `setup/1` macro. The `setup/1` callback runs before every test, in the same process as the test itself. - -Note that we need a mechanism to pass the `bucket` pid from the callback to the test. We do so by using the *test context*. When we return `{:ok, bucket: bucket}` from the callback, ExUnit will merge the second element of the tuple (a dictionary) into the test context. The test context is a map which we can then match in the test definition, providing access to these values inside the block: - -```elixir -test "stores values by key", %{bucket: bucket} do - # `bucket` is now the bucket from the setup block -end -``` - -You can read more about ExUnit cases in the [`ExUnit.Case` module documentation](/docs/stable/ex_unit/ExUnit.Case.html) and more about callbacks in [`ExUnit.Callbacks` docs](/docs/stable/ex_unit/ExUnit.Callbacks.html). - -## 2.4 Other Agent actions - -Besides getting a value and updating the agent state, agents allow us to get a value and update the agent state in one function call via `Agent.get_and_update/2`. Let's implement a `KV.Bucket.delete/2` function that deletes a key from the bucket, returning its current value: - -```elixir -@doc """ -Deletes `key` from `bucket`. - -Returns the current value of `key`, if `key` exists. -""" -def delete(bucket, key) do - Agent.get_and_update(bucket, &HashDict.pop(&1, key)) -end -``` - -Now it is your turn to write a test for the functionality above! Also, be sure to explore the documentation for Agents to learn more about them. - -## 2.5 Client/Server in Agents - -Before we move on to the next chapter, let's discuss the client/server dichotomy in agents. Let's expand the `delete/2` function we have just implemented: - -```elixir -def delete(bucket, key) do - Agent.get_and_update(bucket, fn dict-> - HashDict.pop(dict, key) - end) -end -``` - -Everything that is inside the function we passed to the agent happens in the agent process. In this case, since the agent process is the one receiving and responding to our messages, we say the agent process is the server. Everything outside the function is happening in the client. - -This distinction is important. If there are expensive actions to be done, you must consider if it will be better to perform these actions on the client or on the server. For example: - -```elixir -def delete(bucket, key) do - :timer.sleep(1000) # sleeps the client - Agent.get_and_update(bucket, fn dict -> - :timer.sleep(1000) # sleeps the server - HashDict.pop(dict, key) - end) -end -``` - -When a long action is performed on the server, all other requests to that particular server will wait until the action is done, which may cause some clients to timeout. - -In the next chapter we will explore GenServers, where the segregation between clients and servers is made even more apparent. diff --git a/getting_started/mix_otp/3.markdown b/getting_started/mix_otp/3.markdown deleted file mode 100644 index cfb7100fa..000000000 --- a/getting_started/mix_otp/3.markdown +++ /dev/null @@ -1,277 +0,0 @@ ---- -layout: getting_started -title: 3 GenServer -guide: 3 ---- - -# {{ page.title }} - -
      - -In the previous chapter we used agents to represent our buckets. In the first chapter, we specified we would like to name each bucket so we can do the following: - -```elixir -CREATE shopping -OK - -PUT shopping milk 1 -OK - -GET shopping milk -1 -OK -``` - -Since agents are processes, each bucket has a process identifier (pid) but it doesn't have a name. We have learned about the name registry [in the Process chapter](/getting_started/11.html) and you could be inclined to solve this problem by using such registry. For example, we could create a bucket as: - -```iex -iex> Agent.start_link(fn -> [] end, name: :shopping) -{:ok, #PID<0.43.0>} -iex> KV.Bucket.put(:shopping, "milk", 1) -:ok -iex> KV.Bucket.get(:shopping, "milk") -1 -``` - -However, this a terrible idea! Local names in Elixir must be atoms, which means we would need to convert the bucket name (often received from an external client) to atoms, and **we should never convert user input to atoms**. This is because atoms are not garbage collected. Once an atom is created, it is never reclaimed. Generating atoms from user input would mean the user can inject enough different names to exhaust our system memory! In practice it is more likely you will reach the Erlang VM limit for the maximum number of atoms before you run out of memory, which will bring your system down regardless. - -Instead of abusing the name registry facility, we will instead create our own *registry process* that holds a dictionary that associates the bucket name to the bucket process. - -The registry needs to guarantee the dictionary is always up to date. For example, if one of the bucket processes crashes due to a bug, the registry must clean up the dictionary in order to avoid serving stale entries. In Elixir, we describe this by saying that the registry needs to *monitor* each bucket. - -We will use a [GenServer](/docs/stable/elixir/GenServer.html) to create a registry process that can monitor the bucket process. GenServers are the go-to abstraction for building generic servers in both Elixir and OTP. - -## 3.1 Our first GenServer - -A GenServer is implemented in two parts: the client API and the server callbacks, all in a single module. Create a new file at `lib/kv/registry.ex` with the following contents: - -```elixir -defmodule KV.Registry do - use GenServer - - ## Client API - - @doc """ - Starts the registry. - """ - def start_link(opts \\ []) do - GenServer.start_link(__MODULE__, :ok, opts) - end - - @doc """ - Looks up the bucket pid for `name` stored in `server`. - - Returns `{:ok, pid}` if the bucket exists, `:error` otherwise. - """ - def lookup(server, name) do - GenServer.call(server, {:lookup, name}) - end - - @doc """ - Ensures there is a bucket associated to the given `name` in `server`. - """ - def create(server, name) do - GenServer.cast(server, {:create, name}) - end - - ## Server Callbacks - - def init(:ok) do - {:ok, HashDict.new} - end - - def handle_call({:lookup, name}, _from, names) do - {:reply, HashDict.fetch(names, name), names} - end - - def handle_cast({:create, name}, names) do - if HashDict.get(names, name) do - {:noreply, names} - else - {:ok, bucket} = KV.Bucket.start_link() - {:noreply, HashDict.put(names, name, bucket)} - end - end -end -``` - -The first function is `start_link/1`, which starts a new GenServer passing three arguments: - -1. The module where the server callbacks are implemented, in this case `__MODULE__`, meaning the current module - -2. The initialization arguments, in this case the atom `:ok` - -3. A list of options which can, for example, hold the name of the server - -There are two types of requests you can send to a GenServer: calls and casts. Calls are synchronous and the server **must** send a response back to such requests. Casts are asynchronous and the server won't send a response back. - -The next two functions, `lookup/2` and `create/2` are responsible for sending these requests to the server. The requests are represented by the first argument to `handle_call/3` or `handle_cast/2`. In this case, we have used `{:lookup, name}` and `{:create, name}` respectively. Requests are often specified as tuples, like this, in order to provide more than one "argument" in that first argument slot. It's common to specify the action being requested as the first element of a tuple, and arguments for that action in the remaining elements. - -On the server side, we can implement a variety of callbacks to guarantee the server initialization, termination and handling of requests. Those callbacks are optional and for now we have only implemented the ones we care about. - -The first is the `init/1` callback, that receives the argument given `GenServer.start_link/3` and returns `{:ok, state}`, where state is a new `HashDict`. We can already notice how the `GenServer` API makes the client/server segregation more apparent. `start_link/3` happens in the client, while `init/1` is the respective callback that runs on the server. - -For `call` requests, we must implement a `handle_call/3` callback that receives the `request`, the process from which we received the request (`_from`), and the current server state (`names`). The `handle_call/3` callback returns a tuple in the format `{:reply, reply, new_state}`, where `reply` is what will be sent to the client and the `new_state` is the new server state. - -For `cast` requests, we must implement a `handle_cast/2` callback that receives the `request` and the current server state (`names`). The `handle_cast/2` callback returns a tuple in the format `{:noreply, new_state}`. - -There are other tuple formats both `handle_call/3` and `handle_cast/2` callbacks may return. There are also other callbacks like `terminate/2` and `code_change/3` that we could implement. You are welcome to explore the [full GenServer documentation](/docs/stable/elixir/GenServer.html) to learn more about those. - -For now, let's write some tests to guarantee our GenServer works as expected. - -## 3.2 Testing a GenServer - -Testing a GenServer is not much different from testing an agent. We will spawn the server on a setup callback and use it throughout our tests. Create a file at `test/kv/registry_test.exs` with the following: - -```elixir -defmodule KV.RegistryTest do - use ExUnit.Case, async: true - - setup do - {:ok, registry} = KV.Registry.start_link - {:ok, registry: registry} - end - - test "spawns buckets", %{registry: registry} do - assert KV.Registry.lookup(registry, "shopping") == :error - - KV.Registry.create(registry, "shopping") - assert {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - - KV.Bucket.put(bucket, "milk", 1) - assert KV.Bucket.get(bucket, "milk") == 1 - end -end -``` - -Our test should pass right out of the box! - -To shutdown the registry, we are simply sending a `:shutdown` signal to its process when our test finishes. While this solution is ok for tests, if there is a need to stop a `GenServer` as part of the application logic, it is best to define a `stop/1` function that sends a `call` message causing the server to stop: - -```elixir -## Client API - -@doc """ -Stops the registry. -""" -def stop(server) do - GenServer.call(server, :stop) -end - -## Server Callbacks - -def handle_call(:stop, _from, state) do - {:stop, :normal, :ok, state} -end -``` - -In the example above, the new `handle_call/3` clause is returning the atom `:stop`, along side the reason the server is being stopped (`:normal`), the reply `:ok` and the server state. - -## 3.3 The need for monitoring - -Our registry is almost complete. The only remaining issue is that the registry may become stale if a bucket stops or crashes. Let's add a test to `KV.RegistryTest` that exposes this bug: - -```elixir -test "removes buckets on exit", %{registry: registry} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - Agent.stop(bucket) - assert KV.Registry.lookup(registry, "shopping") == :error -end -``` - -The test above will fail on the last assertion as the bucket name remains in the registry even after we stop the bucket process. - -In order to fix this bug, we need the registry to monitor every bucket it spawns. Once we set up a monitor, the registry will receive a notification every time a bucket exits, allowing us to clean the dictionary up. - -Let's first play with monitors by starting a new console with `iex -S mix`: - -```iex -iex> {:ok, pid} = KV.Bucket.start_link -{:ok, #PID<0.66.0>} -iex> Process.monitor(pid) -#Reference<0.0.0.551> -iex> Agent.stop(pid) -:ok -iex> flush() -{:DOWN, #Reference<0.0.0.551>, :process, #PID<0.66.0>, :normal} -``` - -Note `Process.monitor(pid)` returns a unique reference that allows us to match upcoming messages to that monitoring reference. After we stop the agent, we can `flush()` all messages and notice a `:DOWN` message arrived, with the exact reference returned by monitor, notifying that the bucket process exited with reason `:normal`. - -Let's reimplement the server callbacks to fix the bug and make the test pass. First, we will modify the GenServer state to two dictionaries: one that contains `name -> pid` and another that holds `ref -> name`. Then we need to monitor the buckets on `handle_cast/2` as well as implement a `handle_info/2` callback to handle the monitoring messages. The full server callbacks implementation is shown below: - -```elixir -## Server callbacks - -def init(:ok) do - names = HashDict.new - refs = HashDict.new - {:ok, {names, refs}} -end - -def handle_call({:lookup, name}, _from, {names, _} = state) do - {:reply, HashDict.fetch(names, name), state} -end - -def handle_call(:stop, _from, state) do - {:stop, :normal, :ok, state} -end - -def handle_cast({:create, name}, {names, refs}) do - if HashDict.get(names, name) do - {:noreply, {names, refs}} - else - {:ok, pid} = KV.Bucket.start_link() - ref = Process.monitor(pid) - refs = HashDict.put(refs, ref, name) - names = HashDict.put(names, name, pid) - {:noreply, {names, refs}} - end -end - -def handle_info({:DOWN, ref, :process, _pid, _reason}, {names, refs}) do - {name, refs} = HashDict.pop(refs, ref) - names = HashDict.delete(names, name) - {:noreply, {names, refs}} -end - -def handle_info(_msg, state) do - {:noreply, state} -end -``` - -Observe that we were able to considerably change the server implementation without changing any of the client API. That's one of the benefits of explicitly segregating the server and the client. - -Finally, different from the other callbacks, we have defined a "catch-all" clause for `handle_info/2` that discards any unknown message. To understand why, let's move on to the next section. - -## 3.4 call, cast or info? - -So far we have used three callbacks: `handle_call/3`, `handle_cast/2` and `handle_info/2`. Deciding when to use each is straightforward: - -1. `handle_call/3` must be used for synchronous requests. This should be the default choice as waiting for the server reply is a useful backpressure mechanism. - -2. `handle_cast/2` must be used for asynchronous requests, when you don't care about a reply. A cast does not even guarantee the server has received the message and, for this reason, must be used sparingly. For example, the `create/2` function we have defined in this chapter should have used `call/2`. We have used `cast/2` for didactic purposes. - -3. `handle_info/2` must be used for all other messages a server may receive that are not sent via `GenServer.call/2` or `GenServer.cast/2`, including regular messages sent with `send/2`. The monitoring `:DOWN` messages are a perfect example of this. - -Since any message, including the ones sent via `send/2`, go to `handle_info/2`, there is a chance unexpected messages will arrive to the server. Therefore, if we don't define the `catch-all` clause, those messages could lead our supervisor to crash, because no clause would match. - -We don't need to worry about this for `handle_call/3` and `handle_cast/2` because these requests are only done via the `GenServer` API, so an unknown message is quite likely to be due to a developer mistake. - -## 3.5 Monitors or links? - -We have previously learned about links in the [Process chapter](/getting_started/11.html). Now, with the registry complete, you may be wondering: when should we use monitors and when should we use links? - -Links are bi-directional. If you link two process and one of them crashes, the other side will crash too (unless it is trapping exits). A monitor is uni-directional: only the monitoring process will receive notifications about the the monitored one. Simply put, use links when you want linked crashes, and monitors when you just want to be informed of crashes, exits, and so on. - -Returning to our `handle_cast/2` implementation, you can see the registry is both linking and monitoring the buckets: - -```elixir -{:ok, pid} = KV.Bucket.start_link() -ref = Process.monitor(pid) -``` - -This is a bad idea, as we don't want the registry to crash when a bucket crashes! We will explore solutions to this problem when we talk about supervisors. In a nutshell, we typically avoid creating new processes directly. Instead, we delegate this responsibility to supervisors. As we'll see, supervisors work with links, and that explains why link-based APIs (`spawn_link`, `start_link`, etc) are so prevalent in Elixir and OTP. - -Before jumping into supervisors, let's first explore event managers and event handlers with GenEvent. diff --git a/getting_started/mix_otp/4.markdown b/getting_started/mix_otp/4.markdown deleted file mode 100644 index 4a6a86259..000000000 --- a/getting_started/mix_otp/4.markdown +++ /dev/null @@ -1,209 +0,0 @@ ---- -layout: getting_started -title: 4 GenEvent -guide: 4 ---- - -# {{ page.title }} - -
      - -In this chapter, we will explore GenEvent, another behaviour provided by Elixir and OTP that allows us to spawn an event manager that is able to publish events to many handlers. - -There are two events we are going to emit: one for every time a bucket is added to the registry and another when it is removed from it. - -## 4.1 Event managers - -Let's start a new `iex -S mix` session and explore the GenEvent API a bit: - -```elixir -iex> {:ok, manager} = GenEvent.start_link -{:ok, #PID<0.83.0>} -iex> GenEvent.sync_notify(manager, :hello) -:ok -iex> GenEvent.notify(manager, :world) -:ok -``` - -`GenEvent.start_link/0` starts a new event manager. That is literally all that is required to start a manager. After the manager is created, we can call `GenEvent.notify/2` and `GenEvent.sync_notify/2` to send notifications. - -However, since there are no event handlers tied to the manager, not much happens on every notification. - -Let's create our first handler, still on IEx, that sends all events to a given process: - -```iex -iex> defmodule Forwarder do -...> use GenEvent -...> def handle_event(event, parent) do -...> send parent, event -...> {:ok, parent} -...> end -...> end -iex> GenEvent.add_handler(manager, Forwarder, self()) -:ok -iex> GenEvent.sync_notify(manager, {:hello, :world}) -:ok -iex> flush -{:hello, :world} -:ok -``` - -We created our handler and added it to the manager by calling `GenEvent.add_handler/3` passing: - -1. The manager we previously started and linked -2. The event handler module (named `Forwarder`) we just defined -3. The event handler state: in this case, the current process pid - -After adding the handler, we can see that by calling `sync_notify/2`, the `Forwarder` handler successfully forwards events to our inbox. - -There are a couple things that are important to highlight at this point: - -1. The event handler runs in the same process as the event manager -2. `sync_notify/2` runs event handlers synchronously to the request -3. `notify/2` runs event handlers asynchronously - -Therefore, `sync_notify/2` and `notify/2` are similar to `call/2` and `cast/2` in GenServer and using `sync_notify/2` is generally recommended. It works as a backpressure mechanism in the calling process, to reduce the likelihood of messages being sent more quickly than they can be dispatched to handlers. - -Be sure to check other functionality provided by GenEvent in its [module documentation](/docs/stable/elixir/GenEvent.html). For now we have enough knowledge to add an event manager to our application. - -## 4.2 Registry events - -In order to emit events, we need to change the registry to work with an event manager. While we could automatically start the event manager when the registry is started, for example in the `init/1` callback, it is preferrable to pass the event manager pid/name to `start_link`, decoupling the start of the event manager from the registry. - -Let's first change our tests to showcase the behaviour we want the registry to exhibit. Open up `test/kv/registry_test.exs` and change the existing `setup` callback to the one below, then add the new test: - -```elixir -defmodule Forwarder do - use GenEvent - - def handle_event(event, parent) do - send parent, event - {:ok, parent} - end -end - -setup do - {:ok, manager} = GenEvent.start_link - {:ok, registry} = KV.Registry.start_link(manager) - - GenEvent.add_mon_handler(manager, Forwarder, self()) - {:ok, registry: registry} -end - -test "sends events on create and crash", %{registry: registry} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - assert_receive {:create, "shopping", ^bucket} - - Agent.stop(bucket) - assert_receive {:exit, "shopping", ^bucket} -end -``` - -In order to test the functionality we want to add, we first define a `Forwarder` event handler similar to the one we typed in IEx previously. On `setup`, we start the event manager, pass it as an argument to the registry and add our `Forwarder` handler to the manager so events can be sent to the test process. - -In the test, we create and stop a bucket process and use `assert_receive` to assert we will receive both `:create` and `:exit` messages. `assert_receive` has a default timeout of 500ms which should be more than enough for our tests. Also note that `assert_receive` expects a pattern, rather than a value, that's why we have used `^bucket` to match on the bucket pid. - -Finally, notice we called `GenEvent.add_mon_handler/3` instead of `GenEvent.add_handler/3`. This function adds a handler, as we know, and also tells the event manager to monitor the current process. If the current process dies, the event handler is automatically removed. This makes sense because, in the `Forwarder` case, we should stop forwarding messages if the recipient of those messages (`self()`/the test process) is no longer alive. - -Let's now change the registry to make the tests pass. Open up `lib/kv/registry.ex` and paste the new registry implementation below (comments inlined): - -```elixir -defmodule KV.Registry do - use GenServer - - ## Client API - - @doc """ - Starts the registry. - """ - def start_link(event_manager, opts \\ []) do - # 1. start_link now expects the event manager as argument - GenServer.start_link(__MODULE__, event_manager, opts) - end - - @doc """ - Looks up the bucket pid for `name` stored in `server`. - - Returns `{:ok, pid}` in case a bucket exists, `:error` otherwise. - """ - def lookup(server, name) do - GenServer.call(server, {:lookup, name}) - end - - @doc """ - Ensures there is a bucket associated with the given `name` in `server`. - """ - def create(server, name) do - GenServer.cast(server, {:create, name}) - end - - ## Server callbacks - - def init(events) do - # 2. The init callback now receives the event manager. - # We have also changed the manager state from a tuple - # to a map, allowing us to add new fields in the future - # without needing to rewrite all callbacks. - names = HashDict.new - refs = HashDict.new - {:ok, %{names: names, refs: refs, events: events}} - end - - def handle_call({:lookup, name}, _from, state) do - {:reply, HashDict.fetch(state.names, name), state} - end - - def handle_cast({:create, name}, state) do - if HashDict.get(state.names, name) do - {:noreply, state} - else - {:ok, pid} = KV.Bucket.start_link() - ref = Process.monitor(pid) - refs = HashDict.put(state.refs, ref, name) - names = HashDict.put(state.names, name, pid) - # 3. Push a notification to the event manager on create - GenEvent.sync_notify(state.events, {:create, name, pid}) - {:noreply, %{state | names: names, refs: refs}} - end - end - - def handle_info({:DOWN, ref, :process, pid, _reason}, state) do - {name, refs} = HashDict.pop(state.refs, ref) - names = HashDict.delete(state.names, name) - # 4. Push a notification to the event manager on exit - GenEvent.sync_notify(state.events, {:exit, name, pid}) - {:noreply, %{state | names: names, refs: refs}} - end - - def handle_info(_msg, state) do - {:noreply, state} - end -end -``` - -The changes are straightforward. We now pass the event manager we received as an argument to `start_link` on to `GenServer` initialization. We also change both cast and info callbacks to call `GenEvent.sync_notify/2`. Lastly, we have taken the opportunity to change the server state to a map, making it easier to improve the registry in the future. - -Run the test suite, and all tests should be green again. - -## 4.3 Event streams - -One last functionality worth exploring from `GenEvent` is the ability to consume its events as a stream: - -```elixir -iex> {:ok, manager} = GenEvent.start_link -{:ok, #PID<0.83.0>} -iex> spawn_link fn -> -...> for x <- GenEvent.stream(manager), do: IO.inspect(x) -...> end -:ok -iex> GenEvent.notify(manager, {:hello, :world}) -{:hello, :world} -:ok -``` - -In the example above, we have created a `GenEvent.stream(manager)` that returns a stream (an enumerable) of events that are consumed as they come. Since consuming those events is a blocking action, we spawn a new process that will consume the events and print them to the terminal, and that is exactly the behaviour we see. Every time we call `sync_notify/2` or `notify/2`, the event is printed to the terminal followed by `:ok` (which is just IEx printing the result returned by notify functions). - -Often event streams provide enough functionality for consuming events that we don't need to register our own handlers. However, when custom functionality is required, or during testing, defining our own event handler callbacks is the best way to go. - -At this point, we have an event manager, a registry and potentially many buckets running at the same time. It is about time to start worrying what would happen if any of those processes crash. diff --git a/getting_started/mix_otp/5.markdown b/getting_started/mix_otp/5.markdown deleted file mode 100644 index ac42ccd47..000000000 --- a/getting_started/mix_otp/5.markdown +++ /dev/null @@ -1,355 +0,0 @@ ---- -layout: getting_started -title: 5 Supervisor and Application -guide: 5 ---- - -# {{ page.title }} - -
      - -So far our application requires an event manager and a registry. It may potentially use dozens, if not hundreds, of buckets. While we may think our implementation so far is quite good, no software is bug free, and failures are definitely going to happen. - -When things fail, your first reaction may be: "let's rescue those errors". But, as we have learned in the Getting Started guide, in Elixir we don't have the defensive programming habit of rescuing exceptions, as commonly seen in other languages. Instead, we say "fail fast" or "let it crash". If there is a bug that leads our registry to crash, we have nothing to worry about because we are going to setup a supervisor that will start a fresh copy of the registry. - -In this chapter, we are going to learn about supervisors and also about applications. We are going to create not one, but two supervisors, and use them to supervise our processes. - -## 5.1 Our first Supervisor - -Creating a supervisor is not much different from creating a GenServer. We are going to define a module named `KV.Supervisor`, which will use the [Supervisor](/docs/stable/elixir/Supervisor.html) behaviour, inside the `lib/kv/supervisor.ex` file: - -```elixir -defmodule KV.Supervisor do - use Supervisor - - def start_link do - Supervisor.start_link(__MODULE__, :ok) - end - - @manager_name KV.EventManager - @registry_name KV.Registry - - def init(:ok) do - children = [ - worker(GenEvent, [[name: @manager_name]]), - worker(KV.Registry, [@manager_name, [name: @registry_name]]) - ] - - supervise(children, strategy: :one_for_one) - end -end -``` - -Our supervisor has two children: the event manager and the registry. It's common to give names to processes under supervision so that other processes can access them by name without needing to know their pid. This is useful because a supervised process might crash, in which case its pid will change when the supervisor restarts it. We declare the names of our supervisor's children by using the module attributes `@manager_name` and `@registry_name`, then reference those attributes in the worker definitions. While it's not required that we declare the names of our child processes in module attributes, it's helpful, because doing so helps make them stand out to the reader of our code. - -For example, the `KV.Registry` worker receives two arguments, the first is the name of the event manager and the second is a keyword list of options. In this case, we set the name option to `[name: KV.Registry]` (using our previously-defined module attribute, `@registry_name`), guaranteeing we can access the registry by the name `KV.Registry` throughout the application. It is very common to name the children of a supervisor after the module that defines them, as this association becomes very handy when debugging a live system. - -The order children are declared in the supervisor also matters. Since the registry depends on the event manager, we must start the latter before the former. That's why the `GenEvent` worker must come before the `KV.Registry` worker in the children list. - -Finally, we call `supervise/2`, passing the list of children and the strategy of `:one_for_one`. - -The supervision strategy dictates what happens when one of the children crashes. `:one_for_one` means that if a child dies only one is restarted to replace it. This strategy makes sense for now. If the event manager crashes, there is no reason to restart the registry and vice-versa. However, those dynamics may change once we add more children to supervisor. The `Supervisor` behaviour supports many different strategies and we will discuss three of them in this chapter. - -If we start a console inside our project using `iex -S mix`, we can manually start the supervisor: - -```iex -iex> KV.Supervisor.start_link -{:ok, #PID<0.66.0>} -iex> KV.Registry.create(KV.Registry, "shopping") -:ok -iex> KV.Registry.lookup(KV.Registry, "shopping") -{:ok, #PID<0.70.0>} -``` - -When we started the supervisor tree, both the event manager and registry worker were automatically started, allowing us to create buckets without the need to manually start these processes. - -In practice though, we rarely start the application supervisor manually. Instead it is started as part of the application callback. - -## 5.2 Understanding applications - -We have been working inside an application this entire time. Every time we changed a file and ran `mix compile`, we could see `Generated kv.app` message in the compilation output. - -We can find the generated `.app` file at `_build/dev/lib/kv/ebin/kv.app`. Let's have a look at its contents: - -```erlang -{application,kv, - [{registered,[]}, - {description,"kv"}, - {applications,[kernel,stdlib,elixir,logger]}, - {vsn,"0.0.1"}, - {modules,['Elixir.KV','Elixir.KV.Bucket', - 'Elixir.KV.Registry','Elixir.KV.Supervisor']}]}. -``` - -This file contains Erlang terms (written using Erlang syntax). Even though we are not familiar with Erlang, it is easy to guess this file holds our application definition. It contains our application `version`, all the modules defined by it, as well as a list of applications we depend on, like Erlang's `kernel` and `elixir` itself, and `logger` which is specified in the application list in `mix.exs`. - -It would be pretty boring to update this file manually every time we add a new module to our application. That's why mix generates and maintains it automatically for us. - -We can also configure the generated `.app` file by customizing the values returned by the `application/0` inside our `mix.exs` project file. We will get to that in upcoming chapters. - -### 5.2.1 Starting applications - -When we define an `.app` file, which is the application definition, we are able to start and stop the application as a whole. We haven't worried about this so far for two reasons: - -1. Mix automatically starts our current application for us - -2. Even if Mix didn't start our application for us, our application does not yet need to do anything when it starts - -In any case, let's see how Mix starts the application for us. Let's start a project console with `iex -S mix` and try: - -```iex -iex> Application.start(:kv) -{:error, {:already_started, :kv}} -``` - -Oops, it's already started. - -We can pass an option to mix to ask it to not start our application. Let's give it a try by running `iex -S mix run --no-start`: - -```elixir -iex> Application.start(:kv) -{:error, {:not_started, :logger}} -``` - -Now we get an error because an application that `:kv` depends on (`:logger` in this case) hasn't been started. Mix normally starts the whole hierarchy of applications defined in our project's `mix.exs` file and it does the same for all dependencies if they depend on other applications. But since we passed the `--no-start` flag, we need to either start each application manually in the correct order or call `Application.ensure_all_started` as follows: - -```elixir -iex> Application.ensure_all_started(:kv) -{:ok, [:logger, :kv]} -iex> Application.stop(:kv) -18:12:10.698 [info] Application kv exited :stopped -:ok -``` - -Nothing really exciting happens but it shows how we can control our application. - -> When you run `iex -S mix`, it is equivalent to running `iex -S mix run`. So whenever you need to pass more options to mix when starting iex, it's just a matter of typing `mix run` and then passing any options the `run` command accepts. You can find more information about `run` by running `mix help run` in your shell. - -### 5.2.2 The application callback - -Since we spent all this time talking about how applications are started and stopped, there must be a way to do something useful when the application starts. And indeed, there is! - -We can specify an application callback function. This is a function that will be invoked when the application starts. The function must return a result of `{:ok, pid}`, where `pid` is the process identifier of a supervisor process. - -We can configure the application callback in two steps. First, open up the `mix.exs` file and change `def application` to the following: - -```elixir -def application do - [applications: [], - mod: {KV, []}] -end -``` - -The `:mod` option specifies the "application callback module", followed by the arguments to be passed on application start. The application callback module can be any module that implements the [Application](/docs/stable/elixir/Application.html) behaviour. - -Now that we have specified `KV` as the module callback, we need to change the `KV` module, defined in `lib/kv.ex`: - -```elixir -defmodule KV do - use Application - - def start(_type, _args) do - KV.Supervisor.start_link - end -end -``` - -When we `use Application`, we only need to define a `start/2` function. If we wanted to specify custom behaviour on application stop, we could define a `stop/1` function, as well. In this case, the one automatically defined by `use Application` is fine. - -Let's start our project console once again with `iex -S mix`. We will see a process named `KV.Registry` is already running: - -```iex -iex> KV.Registry.create(KV.Registry, "shopping") -:ok -iex> KV.Registry.lookup(KV.Registry, "shopping") -{:ok, #PID<0.88.0>} -``` - -Excellent! - -### 5.2.3 Projects or applications? - -Mix makes a distinction between projects and applications. Based on the current contents of our `mix.exs` file, we would say we have a Mix project that defines the `:kv` application. As we will see in later chapters, there are projects that don't define any application. - -When we say "project," you should think about Mix. Mix is the tool that manages your project. It knows how to compile your project, test your project and more. It also knows how to compile and start the application relevant to your project. - -When we talk about applications, we talk about OTP. Applications are the entities that are started and stopped as a whole by the runtime. You can learn more about applications in the [docs for the Application module](/docs/stable/elixir/Application.html), as well as by running `mix help compile.app` to learn more about the supported options in `def application`. - -## 5.3 Simple one for one supervisors - -We have now successfully defined our supervisor which is automatically started (and stopped) as part of our application lifecycle. - -Remember however that our `KV.Registry` is both linking and monitoring bucket processes in the `handle_cast/2` callback: - -```elixir -{:ok, pid} = KV.Bucket.start_link() -ref = Process.monitor(pid) -``` - -Links are bi-directional, which implies that a crash in a bucket will crash the registry. Although we now have the supervisor, which guarantees the registry will be back up and running, crashing the registry still means we lose all data associating bucket names to their respective processes. - -In other words, we want the registry to keep on running even if a bucket crashes. Let's write a test: - -```elixir -test "removes bucket on crash", %{registry: registry} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(registry, "shopping") - - # Kill the bucket and wait for the notification - Process.exit(bucket, :shutdown) - assert_receive {:exit, "shopping", ^bucket} - assert KV.Registry.lookup(registry, "shopping") == :error -end -``` - -The test is similar to "removes bucket on exit" except that we are being a bit more harsh. Instead of using `Agent.stop/1`, we are sending an exit signal to shutdown the bucket. Since the bucket is linked to the registry, which is then linked to the test process, killing the bucket causes the registry to crash which then causes the test process to crash too: - -``` -1) test removes bucket on crash (KV.RegistryTest) - test/kv/registry_test.exs:52 - ** (EXIT from #PID<0.94.0>) shutdown -``` - -One possible solution to this issue would be to provide a `KV.Bucket.start/0`, that invokes `Agent.start/1`, and use it from the registry, removing the link between registry and buckets. However, this would be a bad idea, because buckets would not be linked to any process after this change. This means that if someone stops the `kv` application, all buckets would remain alive as they are unreachable. - -We are going to solve this issue by defining a new supervisor that will spawn and supervise all buckets. There is one supervisor strategy, called `:simple_one_for_one`, that is the perfect fit for such situations: it allows us to specify a worker template and supervise many children based on this template. - -Let's define our `KV.Bucket.Supervisor` as follows: - -```elixir -defmodule KV.Bucket.Supervisor do - use Supervisor - - def start_link(opts \\ []) do - Supervisor.start_link(__MODULE__, :ok, opts) - end - - def start_bucket(supervisor) do - Supervisor.start_child(supervisor, []) - end - - def init(:ok) do - children = [ - worker(KV.Bucket, [], restart: :temporary) - ] - - supervise(children, strategy: :simple_one_for_one) - end -end -``` - -There are two changes in this supervisor compared to the first one. - -First, we define a `start_bucket/1` function that will receive a supervisor and start a bucket process as a child of that supervisor. `start_bucket/1` is the function we are going to invoke instead of calling `KV.Bucket.start_link` directly in the registry. - -Second, in the `init/1` callback, we are marking the worker as `:temporary`. This means that if the bucket dies, it won't be restarted! That's because we only want to use the supervisor as a mechanism to group the buckets. The creation of buckets should always pass through the registry. - -Run `iex -S mix` so we can give our new supervisor a try: - -```iex -iex> {:ok, sup} = KV.Bucket.Supervisor.start_link -{:ok, #PID<0.70.0>} -iex> {:ok, bucket} = KV.Bucket.Supervisor.start_bucket(sup) -{:ok, #PID<0.72.0>} -iex> KV.Bucket.put(bucket, "eggs", 3) -:ok -iex> KV.Bucket.get(bucket, "eggs") -3 -``` - -Let's change the registry to work with the buckets supervisor. We are going to follow the same strategy we did with the events manager, where we will explicitly pass the buckets supervisor pid to `KV.Registry.start_link/3`. Let's start by changing the setup callback in `test/kv/registry_test.exs` to do so: - -```elixir -setup do - {:ok, sup} = KV.Bucket.Supervisor.start_link - {:ok, manager} = GenEvent.start_link - {:ok, registry} = KV.Registry.start_link(manager, sup) - - GenEvent.add_mon_handler(manager, Forwarder, self()) - {:ok, registry: registry} -end -``` - -Now let's change the appropriate functions in `KV.Registry` to take the new supervisor into account: - -```elixir -## Client API - -@doc """ -Starts the registry. -""" -def start_link(event_manager, buckets, opts \\ []) do - # 1. Pass the buckets supevisor as argument - GenServer.start_link(__MODULE__, {event_manager, buckets}, opts) -end - -## Server callbacks - -def init({events, buckets}) do - names = HashDict.new - refs = HashDict.new - # 2. Store the buckets supevisor in the state - {:ok, %{names: names, refs: refs, events: events, buckets: buckets}} -end - -def handle_cast({:create, name}, state) do - if HashDict.get(state.names, name) do - {:noreply, state} - else - # 3. Use the buckets supervisor instead of starting buckets directly - {:ok, pid} = KV.Bucket.Supervisor.start_bucket(state.buckets) - ref = Process.monitor(pid) - refs = HashDict.put(state.refs, ref, name) - names = HashDict.put(state.names, name, pid) - GenEvent.sync_notify(state.events, {:create, name, pid}) - {:noreply, %{state | names: names, refs: refs}} - end -end -``` - -Those changes should be enough to make our tests pass! To complete our task, we just need to update our supervisor to also take the buckets supervisor as child. - -## 5.4 Supervision trees - -In order to use the buckets supervisor in our application, we need to add it as a child of `KV.Supervisor`. Notice we are beginning to have supervisors that supervise other supervisors, forming so-called "supervision trees." - -Open up `lib/kv/supervisor.ex`, add an additional module attribute for the buckets supervisor name, and change `init/1` to match the following: - -```elixir -@manager_name KV.EventManager -@registry_name KV.Registry -@bucket_sup_name KV.Bucket.Supervisor - -def init(:ok) do - children = [ - worker(GenEvent, [[name: @manager_name]]), - supervisor(KV.Bucket.Supervisor, [[name: @bucket_sup_name]]), - worker(KV.Registry, [@manager_name, @bucket_sup_name, [name: @registry_name]]) - ] - - supervise(children, strategy: :one_for_one) -end -``` - -This time we have added a supervisor as child and given it the name of `KV.Bucket.Supervisor` (again, the same name as the module). We have also updated the `KV.Registry` worker to receive the bucket supervisor name as argument. - -Also remember that the order in which children are declared is important. Since the registry depends on the buckets supervisor, the buckets supervisor must be listed before it in the children list. - -Since we have added more children to the supervisor, it is important to evaluate if the `:one_for_one` strategy is still correct. One flaw that shows up right away is the relationship between registry and buckets supervisor. If the registry dies, the buckets supervisor must die too, because once the registry dies all information linking the bucket name to the bucket process is lost. If the buckets supervisor is kept alive, it would be impossible to reach those buckets. - -We could consider moving to another strategy like `:one_for_all`. The `:one_for_all` strategy kills and restarts all children whenever one of the children die. This change is not ideal either, because a crash in the registry should not crash the event manager. In fact, doing so would be harmful, as crashing the event manager would cause all installed event handlers to be removed. - -One possible solution to this problem is to create another supervisor that will supervise the registry and buckets supervisor with `:one_for_all` strategy, and have the root supervisor supervise both the event manager and the new supervisor with `:one_for_one` strategy. The proposed tree would have the following format: - -``` -* root supervisor [one_for_one] - * event manager - * supervisor [one_for_all] - * buckets supervisor [simple_one_for_one] - * buckets - * registry -``` - -You can take a shot at building this new supervision tree, but we will stop here. This is because in the next chapter we will make changes to the registry that will allow the registry data to be persisted, making the `:one_for_one` strategy a perfect fit. - -Remember, there are other strategies and other options that could be given to `worker/2`, `supervisor/2` and `supervise/2` functions, so don't forget to check out [the Supervisor module documentation](/docs/stable/elixir/Supervisor.html). diff --git a/getting_started/mix_otp/6.markdown b/getting_started/mix_otp/6.markdown deleted file mode 100644 index 480f96a49..000000000 --- a/getting_started/mix_otp/6.markdown +++ /dev/null @@ -1,475 +0,0 @@ ---- -layout: getting_started -title: 6 ETS -guide: 6 ---- - -# {{ page.title }} - -
      - -Every time we need to look up a bucket, we need to send a message to the registry. In some applications, this means the registry may become a bottleneck! - -In this chapter we will learn about ETS (Erlang Term Storage), and how to use it as a cache mechanism. Later we will expand its usage to persist data from the supervisor to its children, allowing data to persist even on crashes. - -> Warning! Don't use ETS as a cache prematurely! Log and analyze your application performance and identify which parts are bottlenecks, so you know *whether* you should cache, and *what* you should cache. This chapter is merely an example of how ETS can be used, once you've determined the need. - -## 6.1 ETS as a cache - -ETS allows us to store any Erlang/Elixir term in an in-memory table. Working with ETS tables is done via [erlang's `:ets` module](http://www.erlang.org/doc/man/ets.html): - -```iex -iex> table = :ets.new(:buckets_registry, [:set, :protected]) -8207 -iex> :ets.insert(table, {"foo", self}) -true -iex> :ets.lookup(table, "foo") -[{"foo", #PID<0.41.0>}] -``` - -When creating an ETS table, two arguments are required: the table name and a set of options. From the available options, we passed the table type and its access rules. We have chosen the `:set` type, which means that keys cannot be duplicated. We've also set the table's access to `:protected`, which means that only the process that created the table can write to it, but all processes can read it from it. Those are actually the default values, so we will skip them from now on. - -ETS tables can also be named, allowing us to access them by a given name: - -```iex -iex> :ets.new(:buckets_registry, [:named_table]) -:buckets_registry -iex> :ets.insert(:buckets_registry, {"foo", self}) -true -iex> :ets.lookup(:buckets_registry, "foo") -[{"foo", #PID<0.41.0>}] -``` - -Let's change the `KV.Registry` to use ETS tables. We will use the same technique as we did for the event manager and buckets supervisor, and pass the ETS table name explicitly on `start_link`. Remember that, as with server names, any local process that knows an ETS table name will be able to access that table. - -Open up `lib/kv/registry.ex`, and let's change its implementation. We've added comments to the source code to highlight the changes we've made: - -```elixir -defmodule KV.Registry do - use GenServer - - ## Client API - - @doc """ - Starts the registry. - """ - def start_link(table, event_manager, buckets, opts \\ []) do - # 1. We now expect the table as argument and pass it to the server - GenServer.start_link(__MODULE__, {table, event_manager, buckets}, opts) - end - - @doc """ - Looks up the bucket pid for `name` stored in `table`. - - Returns `{:ok, pid}` if a bucket exists, `:error` otherwise. - """ - def lookup(table, name) do - # 2. lookup now expects a table and looks directly into ETS. - # No request is sent to the server. - case :ets.lookup(table, name) do - [{^name, bucket}] -> {:ok, bucket} - [] -> :error - end - end - - @doc """ - Ensures there is a bucket associated with the given `name` in `server`. - """ - def create(server, name) do - GenServer.cast(server, {:create, name}) - end - - ## Server callbacks - - def init({table, events, buckets}) do - # 3. We have replaced the names HashDict by the ETS table - ets = :ets.new(table, [:named_table, read_concurrency: true]) - refs = HashDict.new - {:ok, %{names: ets, refs: refs, events: events, buckets: buckets}} - end - - # 4. The previous handle_call callback for lookup was removed - - def handle_cast({:create, name}, state) do - # 5. Read and write to the ETS table instead of the HashDict - case lookup(state.names, name) do - {:ok, _pid} -> - {:noreply, state} - :error -> - {:ok, pid} = KV.Bucket.Supervisor.start_bucket(state.buckets) - ref = Process.monitor(pid) - refs = HashDict.put(state.refs, ref, name) - :ets.insert(state.names, {name, pid}) - GenEvent.sync_notify(state.events, {:create, name, pid}) - {:noreply, %{state | refs: refs}} - end - end - - def handle_info({:DOWN, ref, :process, pid, _reason}, state) do - # 6. Delete from the ETS table instead of the HashDict - {name, refs} = HashDict.pop(state.refs, ref) - :ets.delete(state.names, name) - GenEvent.sync_notify(state.events, {:exit, name, pid}) - {:noreply, %{state | refs: refs}} - end - - def handle_info(_msg, state) do - {:noreply, state} - end -end -``` - -Notice that before our changes `KV.Registry.lookup/2` sent requests to the server, but now it reads directly from the ETS table, which is shared across all processes. That's the main idea behind the cache mechanism we are implementing. - -In order for the cache mechanism to work, the created ETS table needs to have access `:protected` (the default), so all clients can read from it, while only the `KV.Registry` process writes to it. We have also set `read_concurrency: true` when starting the table, optimizing the table for the common scenario of concurrent read operations. - -The changes we have performed above have definitely broken our tests. For starters, there is a new argument we need to pass to `KV.Registry.start_link/3`. Let's start amending our tests in `test/kv/registry_test.exs` by rewriting the `setup` callback: - -```elixir -setup do - {:ok, sup} = KV.Bucket.Supervisor.start_link - {:ok, manager} = GenEvent.start_link - {:ok, registry} = KV.Registry.start_link(:registry_table, manager, sup) - - GenEvent.add_mon_handler(manager, Forwarder, self()) - {:ok, registry: registry, ets: :registry_table} -end -``` - -Notice we are passing the table name of `:registry_table` to `KV.Registry.start_link/3` as well as returning `ets: :registry_table` as part of the test context. - -After changing the callback above, we will still have failures in our test suite. All in the format of: - -``` -1) test spawns buckets (KV.RegistryTest) - test/kv/registry_test.exs:38 - ** (ArgumentError) argument error - stacktrace: - (stdlib) :ets.lookup(#PID<0.99.0>, "shopping") - (kv) lib/kv/registry.ex:22: KV.Registry.lookup/2 - test/kv/registry_test.exs:39 -``` - -This is happening because we are passing the registry pid to `KV.Registry.lookup/2` while now it expects the ETS table. We can fix this by changing all occurrences of: - -```elixir -KV.Registry.lookup(registry, ...) -``` - -to: - -```elixir -KV.Registry.lookup(ets, ...) -``` - -Where `ets` will be retrieved in the same way we retrieve the registry: - -```elixir -test "spawns buckets", %{registry: registry, ets: ets} do -``` - -Let's change our tests to pass `ets` to `lookup/2`. Once we finish these changes, some tests will continue to fail. You may even notice tests pass and fail inconsistently between runs. For example, the "spawns buckets" test: - -```elixir -test "spawns buckets", %{registry: registry, ets: ets} do - assert KV.Registry.lookup(ets, "shopping") == :error - - KV.Registry.create(registry, "shopping") - assert {:ok, bucket} = KV.Registry.lookup(ets, "shopping") - - KV.Bucket.put(bucket, "milk", 1) - assert KV.Bucket.get(bucket, "milk") == 1 -end -``` - -may be failing on this line: - -```elixir -assert {:ok, bucket} = KV.Registry.lookup(ets, "shopping") -``` - -However how can this line fail if we just created the bucket in the previous line? - -The reason those failures are happening is because, for didactic purposes, we have made two mistakes: - -1. We are prematurely optimizing (by adding this cache layer) -2. We are using `cast/2` (while we should be using `call/2`) - -## 6.2 Race conditions? - -Developing in Elixir does not make your code free of race conditions. However, Elixir's simple abstractions where nothing is shared by default make it easier to spot a race condition's root cause. - -What is happening in our test is that there is a delay in between an operation and the time we can observe this change in the ETS table. Here is what we were expecting to happen: - -1. We invoke `KV.Registry.create(registry, "shopping")` -2. The registry creates the bucket and updates the cache table -3. We access the information from the table with `KV.Registry.lookup(ets, "shopping")` -4. The command above returns `{:ok, bucket}` - -However, since `KV.Registry.create/2` is a cast operation, the command will return before we actually write to the table! In other words, this is happening: - -1. We invoke `KV.Registry.create(registry, "shopping")` -2. We access the information from the table with `KV.Registry.lookup(ets, "shopping")` -3. The command above returns `:error` -4. The registry creates the bucket and updates the cache table - -To fix the failure we just need to make `KV.Registry.create/2` synchronous by using `call/2` rather than `cast/2`. This will guarantee that the client will only continue after changes have been made to the table. Let's change the function and its callback as follows: - -```elixir -def create(server, name) do - GenServer.call(server, {:create, name}) -end - -def handle_call({:create, name}, _from, state) do - case lookup(state.names, name) do - {:ok, pid} -> - {:reply, pid, state} # Reply with pid - :error -> - {:ok, pid} = KV.Bucket.Supervisor.start_bucket(state.buckets) - ref = Process.monitor(pid) - refs = HashDict.put(state.refs, ref, name) - :ets.insert(state.names, {name, pid}) - GenEvent.sync_notify(state.events, {:create, name, pid}) - {:reply, pid, %{state | refs: refs}} # Reply with pid - end -end -``` - -We simply changed the callback from `handle_cast/2` to `handle_call/3` and changed it to reply with the pid of the created bucket. - -Let's run the tests once again. This time though, we will pass the `--trace` option: - - $ mix test --trace - -The `--trace` option is useful when your tests are deadlocking or there are race conditions, as it runs all tests synchronously (`async: true` has no effect) and shows detailed information about each test. This time we should be down to one failure (that may be intermittent): - -``` -1) test removes buckets on exit (KV.RegistryTest) - test/kv/registry_test.exs:48 - Assertion with == failed - code: KV.Registry.lookup(ets, "shopping") == :error - lhs: {:ok, #PID<0.103.0>} - rhs: :error - stacktrace: - test/kv/registry_test.exs:52 -``` - -According to the failure message, we are expecting that the bucket no longer exists on the table, but it still does! This problem is the opposite of the one we have just solved: while previously there was a delay between the command to create a bucket and updating the table, now there is a delay between the bucket process dying and its entry being removed from the table. - -Unfortunately this time we cannot simply change `handle_info/2` to a synchronous operation. We can, however, fix our tests by using event manager notifications. Let's take another look at our `handle_info/2` implementation: - -```elixir -def handle_info({:DOWN, ref, :process, pid, _reason}, state) do - # 5. Delete from the ETS table instead of the HashDict - {name, refs} = HashDict.pop(state.refs, ref) - :ets.delete(state.names, name) - GenEvent.sync_notify(state.event, {:exit, name, pid}) - {:noreply, %{state | refs: refs}} -end -``` - -Notice that we are deleting from the ETS table **before** we send the notification. This is by design! This means that when we receive the `{:exit, name, pid}` notification, the table will already be up to date. Let's update the remaining failing test as follows: - -```elixir -test "removes buckets on exit", %{registry: registry, ets: ets} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(ets, "shopping") - Agent.stop(bucket) - assert_receive {:exit, "shopping", ^bucket} # Wait for event - assert KV.Registry.lookup(ets, "shopping") == :error -end -``` - -We have simply amended the test to guarantee we first receive the `{:exit, name, pid}` message before invoking `KV.Registry.lookup/2`. - -It is important to observe that we were able to keep our suite passing without a need to use `:timer.sleep/1` or other tricks. Most of the time, we can rely on events, monitoring and messages to assert the system is in an expected state before performing assertions. - -For your convenience, here is the fully passing test case: - -```elixir -defmodule KV.RegistryTest do - use ExUnit.Case, async: true - - defmodule Forwarder do - use GenEvent - - def handle_event(event, parent) do - send parent, event - {:ok, parent} - end - end - - setup do - {:ok, sup} = KV.Bucket.Supervisor.start_link - {:ok, manager} = GenEvent.start_link - {:ok, registry} = KV.Registry.start_link(:registry_table, manager, sup) - - GenEvent.add_mon_handler(manager, Forwarder, self()) - {:ok, registry: registry, ets: :registry_table} - end - - test "sends events on create and crash", %{registry: registry, ets: ets} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(ets, "shopping") - assert_receive {:create, "shopping", ^bucket} - - Agent.stop(bucket) - assert_receive {:exit, "shopping", ^bucket} - end - - test "spawns buckets", %{registry: registry, ets: ets} do - assert KV.Registry.lookup(ets, "shopping") == :error - - KV.Registry.create(registry, "shopping") - assert {:ok, bucket} = KV.Registry.lookup(ets, "shopping") - - KV.Bucket.put(bucket, "milk", 1) - assert KV.Bucket.get(bucket, "milk") == 1 - end - - test "removes buckets on exit", %{registry: registry, ets: ets} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(ets, "shopping") - Agent.stop(bucket) - assert_receive {:exit, "shopping", ^bucket} # Wait for event - assert KV.Registry.lookup(ets, "shopping") == :error - end - - test "removes bucket on crash", %{registry: registry, ets: ets} do - KV.Registry.create(registry, "shopping") - {:ok, bucket} = KV.Registry.lookup(ets, "shopping") - - # Kill the bucket and wait for the notification - Process.exit(bucket, :shutdown) - assert_receive {:exit, "shopping", ^bucket} - assert KV.Registry.lookup(ets, "shopping") == :error - end -end -``` - -With tests passing, we just need to update the supervisor `init/1` callback at `lib/kv/supervisor.ex` to pass the ETS table name as an argument to the registry worker: - -```elixir -@manager_name KV.EventManager -@registry_name KV.Registry -@ets_registry_name KV.Registry -@bucket_sup_name KV.Bucket.Supervisor - -def init(:ok) do - children = [ - worker(GenEvent, [[name: @manager_name]]), - supervisor(KV.Bucket.Supervisor, [[name: @bucket_sup_name]]), - worker(KV.Registry, [@ets_registry_name, @manager_name, - @bucket_sup_name, [name: @registry_name]]) - ] - - supervise(children, strategy: :one_for_one) -end -``` - -Note that we are using `KV.Registry` as name for the ETS table as well, which makes it convenient to debug, as it points to the module using it. ETS names and process names are stored in different registries, so there is no chance of conflicts. - -## 6.3 ETS as persistent storage - -So far we have created an ETS table during the registry initialization but we haven't bothered to close the table on registry termination. That's because the ETS table is "linked" (in a figure of speech) to the process that creates it. If that process dies, the table is automatically closed. - -This is extremely convenient as a default behaviour, and we can use it even more to our advantage. Remember that there is a dependency between the registry and the buckets supervisor. If the registry dies, we want the buckets supervisor to die too, because once the registry dies all information linking the bucket name to the bucket process is lost. However, what if we could keep the registry data even if the registry process crashes? If we are able to do so, we remove the dependency between the registry and the buckets supervisor, making the `:one_for_one` strategy the perfect strategy for our supevisor. - -A couple of changes will be required in order to make this happen. First, we'll need to start the ETS table inside the supervisor. Second, we'll need to change the table's access type from `:protected` to `:public`, because the owner is the supervisor, but the process doing the writes is still the manager. - -Let's get started by first changing `KV.Supervisor`'s `init/1` callback: - -```elixir -def init(:ok) do - ets = :ets.new(@ets_registry_name, - [:set, :public, :named_table, {:read_concurrency, true}]) - - children = [ - worker(GenEvent, [[name: @manager_name]]), - supervisor(KV.Bucket.Supervisor, [[name: @bucket_sup_name]]), - worker(KV.Registry, [ets, @manager_name, - @bucket_sup_name, [name: @registry_name]]) - ] - - supervise(children, strategy: :one_for_one) -end -``` - -Next, we change `KV.Registry`'s `init/1` callback, as it no longer needs to create a table. It should instead just use the one given as an argument: - -```elixir -def init({table, events, buckets}) do - refs = HashDict.new - {:ok, %{names: table, refs: refs, events: events, buckets: buckets}} -end -``` - -Finally, we just need to change the `setup` callback in `test/kv/registry_test.exs` to explicitly create the ETS table. We will use this opportunity to also split the `setup` functionality into a private function that will be handy soon: - -```elixir -setup do - ets = :ets.new(:registry_table, [:set, :public]) - registry = start_registry(ets) - {:ok, registry: registry, ets: ets} -end - -defp start_registry(ets) do - {:ok, sup} = KV.Bucket.Supervisor.start_link - {:ok, manager} = GenEvent.start_link - {:ok, registry} = KV.Registry.start_link(ets, manager, sup) - - GenEvent.add_mon_handler(manager, Forwarder, self()) - registry -end -``` - -After those changes, our test suite should continue to be green! - -There is just one last scenario to consider: once we receive the ETS table, there may be existing bucket pids on the table. After all, that's the whole purpose of this change! However, the newly started registry is not monitoring those buckets, as they were created as part of previous, now defunct, registry. This means that the table may go stale, because we won't remove those buckets if they die. - -Let's add a test to `test/kv/registry_test.exs` that shows this bug: - -```elixir -test "monitors existing entries", %{registry: registry, ets: ets} do - bucket = KV.Registry.create(registry, "shopping") - - # Kill the registry. We unlink first, otherwise it will kill the test - Process.unlink(registry) - Process.exit(registry, :shutdown) - - # Start a new registry with the existing table and access the bucket - start_registry(ets) - assert KV.Registry.lookup(ets, "shopping") == {:ok, bucket} - - # Once the bucket dies, we should receive notifications - Process.exit(bucket, :shutdown) - assert_receive {:exit, "shopping", ^bucket} - assert KV.Registry.lookup(ets, "shopping") == :error -end -``` - -Run the new test and it will fail with: - -``` -1) test monitors existing entries (KV.RegistryTest) - test/kv/registry_test.exs:72 - No message matching {:exit, "shopping", ^bucket} - stacktrace: - test/kv/registry_test.exs:85 -``` - -That's what we expected. If the bucket is not being monitored, the registry is not notified when it dies and therefore no event is sent. We can fix this by changing `KV.Registry`'s `init/1` callback one last time to setup monitors for all existing entries in the table: - -```elixir -def init({table, events, buckets}) do - refs = :ets.foldl(fn {name, pid}, acc -> - HashDict.put(acc, Process.monitor(pid), name) - end, HashDict.new, table) - - {:ok, %{names: table, refs: refs, events: events, buckets: buckets}} -end -``` - -We use `:ets.foldl/3` to go through all entries in the table, similar to `Enum.reduce/3`, invoking the given function for each element in the table with the given accumulator. In the function callback, we monitor each pid in the table and update the refs dictionary accordingly. If any of the entries is already dead, we will still receive the `:DOWN` message, causing them to be purged later. - -In this chapter we were able to make our application more robust by using an ETS table that is owned by the supervisor and passed to the registry. We have also explored how to use ETS as a cache and discussed some of the race conditions we may run into as data becomes shared between the server and all clients. diff --git a/getting_started/mix_otp/7.markdown b/getting_started/mix_otp/7.markdown deleted file mode 100644 index d94ce213a..000000000 --- a/getting_started/mix_otp/7.markdown +++ /dev/null @@ -1,269 +0,0 @@ ---- -layout: getting_started -title: 7 Dependencies and umbrella projects -guide: 7 ---- - -# {{ page.title }} - -
      - -In this chapter, we will briefly discuss how to manage dependencies in Mix. - -Our `kv` application is complete, so it's time to implement the server that will handle the requests we defined in the first chapter: - -``` -CREATE shopping -OK - -PUT shopping milk 1 -OK - -PUT shopping eggs 3 -OK - -GET shopping milk -1 -OK - -DELETE shopping eggs -OK -``` - -However, instead of adding more code to the `kv` application, we are going to build the TCP server as another application that is a client of the `kv` application. Since the whole runtime and Elixir ecosystem are geared towards applications, it makes sense to break our projects into smaller applications that work together rather than building a big, monolithic app. - -Before creating our new application, we must discuss how Mix handles dependencies. In practice, there are two kinds of dependencies we usually work with: internal and external dependencies. Mix supports mechanisms to work with both of them. - -## 7.1 External dependencies - -External dependencies are the ones not tied to your business domain. For example, if you need a HTTP API for your distributed KV application, you can use the [Plug](http://github.com/elixir-lang/plug) project as an external dependency. - -Installing external dependencies is simple. Most commonly, we use the [Hex Package Manager](http://hex.pm), by listing the dependency inside the deps function in our `mix.exs` file: - -```elixir -def deps do - [{:plug, "~> 0.5.0"}] -end -``` - -This dependency refers to the latest version of plug in the 0.5.x version series that has been pushed to Hex. This is indicated by the `~>` preceding the version number. For more information on specifying version requirements, see the [documentation for the Version module](http://elixir-lang.org/docs/stable/elixir/Version.html). - -Typically, stable releases are pushed to Hex. If you want to depend on an external dependency still in development, Mix is able to manage git dependencies, too: - -```elixir -def deps do - [{:plug, git: "git://github.com/elixir-lang/plug.git"}] -end -``` - -You will notice that when you add a dependency to your project, Mix generates a `mix.lock` file that guarantees *repeatable builds*. The lock file must be checked in to your version control system, to guarantee that everyone who uses the project will use the same dependency versions as you. - -Mix provides many tasks for working with dependencies, which can be seen in `mix help`: - - $ mix help - mix deps # List dependencies and their status - mix deps.clean # Remove the given dependencies' files - mix deps.compile # Compile dependencies - mix deps.get # Get all out of date dependencies - mix deps.unlock # Unlock the given dependencies - mix deps.update # Update the given dependencies - -The most common tasks are `mix deps.get` and `mix deps.update`. Once fetched, dependecies are automatically compiled for you. You can read more about deps by typing `mix help deps`, and in the [documentation for the Mix.Tasks.Deps module](http://elixir-lang.org/docs/stable/mix/Mix.Tasks.Deps.html). - -## 7.2 Internal dependencies - -Internal dependencies are the ones that are specific to your project. They usually don't make sense outside the scope of your project/company/organization. Most of the time, you want to keep them private, whether due to technical, economic or business reasons. - -If you have an internal dependency, Mix supports two methods of working with them: git repositories or umbrella projects. - -For example, if you push the `kv` project to a git repository, you just need to list it in your deps code in order to use it: - -```elixir -def deps do - [{:kv, git: "git://github.com/YOUR_ACCOUNT/kv.git"}] -end -``` - -It doesn't matter if the git repository is public or private, Mix will be able to fetch it for you as long as you have the proper credentials. - -However, using git dependencies for internal dependencies is somewhat discouraged in Elixir. Remember that the runtime and the Elixir ecosystem already provide the concept of applications. As such, we expect you to frequently break your code into applications that can be organized logically, even within a single project. - -However, if you push every application as a separate project to a git repository, your projects can become very hard to maintain, because now you will have to spend a lot of time managing those git repositories rather than writing your code. - -For this reason, Mix supports "umbrella projects." Umbrella projects allow you to create one project that hosts many applications and push all of them to a single git repository. That is exactly the style we are going to explore in the next sections. - -What we are going to do is create a new mix project. We are going to creatively name it `kv_umbrella`, and this new project will have both the existing `kv` application and the new `kv_server` application inside. The directory structure will look like this: - - + kv_umbrella - + apps - + kv - + kv_server - -The interesting thing about this approach is that Mix has many conveniences for working with such projects, such as the ability to compile and test all applications inside `apps` with a single command. However, even though they are all listed together inside `apps`, they are still decoupled from each other, so you can build, test and deploy each application in isolation if you want to. - -So let's get started! - -## 7.3 Umbrella projects - -Let's start a new project using `mix new`. This new project will be named `kv_umbrella` and we need to pass the `--umbrella` option when creating it. Do not create this new project inside the existing `kv` project! - - $ mix new kv_umbrella --umbrella - * creating .gitignore - * creating README.md - * creating mix.exs - * creating apps - * creating config - * creating config/config.exs - -From the printed information, we can see far fewer files are generated. The generated `mix.exs` file is different too. Let's take a look (comments have been removed): - -```elixir -defmodule KvUmbrella.Mixfile do - use Mix.Project - - def project do - [apps_path: "apps", - deps: deps] - end - - defp deps do - [] - end -end -``` - -What makes this project different from the previous one is simply the `apps_path: "apps"` entry in the project definition. This means this project will act as an umbrella. Such projects do not have source files nor tests, although they can have dependencies which are only available for themselves. We'll create new application projects inside the apps directory. We call these applications "umbrella children". - -Let's move inside the apps directory and start building `kv_server`. This time, we are going to pass the `--sup` flag, which will tell Mix to generate a supervision tree automatically for us, instead of building one manually as we did in previous chapters: - - $ cd kv_umbrella/apps - $ mix new kv_server --module KVServer --sup - -The generated files are similar to the ones we first generated for `kv`, with a few differences. Let's open up `mix.exs`: - -```elixir -defmodule KVServer.Mixfile do - use Mix.Project - - def project do - [app: :kv_server, - version: "0.0.1", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 0.14.1-dev", - deps: deps] - end - - def application do - [applications: [:logger], - mod: {KVServer, []}] - end - - defp deps do - [] - end -end -``` - -First of all, since we generated this project inside `kv_umbrella/apps`, Mix automatically detected the umbrella structure and added two lines to the project definition: - -```elixir -deps_path: "../../deps", -lockfile: "../../mix.lock", -``` - -Those options mean all dependencies will be checked out to `kv_umbrella/deps`, and they will share the same lock file. Those two lines are saying that if two applications in the umbrella share the same dependency, they won't be fetched twice. They'll be fetched once, and Mix will ensure that both apps are always running against the same version of their shared dependency. - -The second change is in the `application` function inside `mix.exs`: - -```elixir -def application do - [applications: [:logger], - mod: {KVServer, []}] -end -``` - -Because we passed the `--sup` flag, Mix automatically added `mod: {KVServer, []}`, specifying that `KVServer` is our application callback module. `KVServer` will start our application supervision tree. - -In fact, let's open up `lib/kv_server.ex`: - -```elixir -defmodule KVServer do - use Application - - def start(_type, _args) do - import Supervisor.Spec, warn: false - - children = [ - # worker(KVServer.Worker, [arg1, arg2, arg3]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) - end -end -``` - -Notice that it defines the application callback function, `start/2`, and instead of defining a supervisor named `KVServer.Supervisor` that uses the `Supervisor` module, it conveniently defined the supervisor inline! You can read more about such supervisors by reading [the Supervisor module documentation](/docs/stable/elixir/Supervisor.html). - -We can already try out our first umbrella child. We could run tests inside the `apps/kv_server` directory, but that wouldn't be much fun. Instead, go to the root of the umbrella project and run `mix test`: - - $ mix test - -And it works! - -Since we want `kv_server` to eventually use the functionality we defined in `kv`, we need to add `kv` as a dependency to our application. - -## 7.4 In umbrella dependencies - -Mix supports an easy mechanism to make one umbrella child depend on another. Open up `apps/kv_server/mix.exs` and change the `deps/0` function to the following: - -```elixir -defp deps do - [{:kv, in_umbrella: true}] -end -``` - -The line above makes `:kv` available as a dependency inside `:kv_server`. We can invoke the modules defined in `:kv` but it does not automatically start the `:kv` application. For that, we also need to list `:kv` as an application inside `application/0`: - -```elixir -def application do - [applications: [:logger, :kv], - mod: {KVServer, []}] -end -``` - -Now Mix will guarantee the `:kv` application is started before `:kv_server` is started. - -Finally, copy the `kv` application we have built so far to the `apps` directory in our new umbrella project. The final directory structure should match the structure we mentioned earlier: - - + kv_umbrella - + apps - + kv - + kv_server - -We now just need to modify `apps/kv/mix.exs` to contain the umbrella entries we have seen in `apps/kv_server/mix.exs`. Open up `apps/kv/mix.exs` and add to the `project` function: - -```elixir -deps_path: "../../deps", -lockfile: "../../mix.lock", -``` - -Now you can run tests for both projects from the umbrella root with `mix test`. Sweet! - -Remember that umbrella projects are a convenience to help you organize and manage your applications. Applications inside the `apps` directory are still decoupled from each other. Each application has its independent configuration, and dependencies in between them must be explicitly listed. This allows them to be developed together, but compiled, tested and deployed independently if desired. - -## 7.5 Summing up - -In this chapter we have learned more about Mix dependencies and umbrella projects. We have decided to build an umbrella project because we consider `kv` and `kv_server` to be internal dependencies that matter only in the context of this project. - -In the future, you are going to write applications and you will notice they can be easily extracted into a concise unit that can be used by different projects. In such cases, using Git or Hex dependencies is the way to go. - -Here are a couple questions you can ask yourself when working with dependencies. Start with: does this application makes sense outside this project? - -* If no, use an umbrella project with umbrella children. -* If yes, can this project be shared outside your company / organization? - * If no, use a private git repository. - * If yes, push your code to a git repository and do frequent releases using [Hex](http://hex.pm). - -With our umbrella project up and running, it is time to start writing our server. diff --git a/getting_started/mix_otp/8.markdown b/getting_started/mix_otp/8.markdown deleted file mode 100644 index 5cce975b0..000000000 --- a/getting_started/mix_otp/8.markdown +++ /dev/null @@ -1,294 +0,0 @@ ---- -layout: getting_started -title: 8 Task and gen_tcp -guide: 8 ---- - -# {{ page.title }} - -
      - -In this chapter, we are going to learn how to use [Erlang's `:gen_tcp` module](http://erlang.org/doc/man/gen_tcp.html) to serve requests. In future chapters we will expand our server so it can actually serve the commands. This will also provide a great opportunity to explore Elixir's `Task` module. - -## 8.1 Echo server - -We will start our TCP server by first implementing an echo server. It will simply send a response with the text it received in the request. We will slowly improve our server until it is supervised and ready to handle multiple connections. - -A TCP server, in broad strokes, performs the following steps: - -1. Listens to a port until the port is available and it gets hold of the socket -2. Waits for a client connection on that port and accepts it -3. Reads the client request and writes a response back - -Let's implement those steps. Move to the `apps/kv_server` application, open up `lib/kv_server.ex`, and add the following functions: - -```elixir -def accept(port) do - # The options below mean: - # - # 1. `:binary` - receives data as binaries (instead of lists) - # 2. `packet: :line` - receives data line by line - # 3. `active: false` - block on `:gen_tcp.recv/2` until data is available - # - {:ok, socket} = :gen_tcp.listen(port, - [:binary, packet: :line, active: false]) - IO.puts "Accepting connections on port #{port}" - loop_acceptor(socket) -end - -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - serve(client) - loop_acceptor(socket) -end - -defp serve(client) do - client - |> read_line() - |> write_line(client) - - serve(client) -end - -defp read_line(socket) do - {:ok, data} = :gen_tcp.recv(socket, 0) - data -end - -defp write_line(line, socket) do - :gen_tcp.send(socket, line) -end -``` - -We are going to start our server by calling `KVServer.accept(4040)`, where 4040 is the port. The first step in `accept/1` is to listen to the port until the socket becomes available and then call `loop_acceptor/1`. `loop_acceptor/1` is just a loop accepting client connections. For each accepted connection, we call `serve/1`. - -`serve/1` is another loop that reads a line from the socket and writes those lines back to the socket. Note that the `serve/1` function uses [the pipeline operator `|>`](/docs/stable/elixir/Kernel.html#|>/2) to express this flow of operations. The pipeline operator evaluates the left side and passes its result as first argument to the function on the right side. The example above: - -```elixir -socket |> read_line() |> write_line(socket) -``` - -is equivalent to: - -```elixir -write_line(read_line(socket), socket) -``` - -> When using the `|>` operator, it is important to add parentheses to the function calls due to how operator precedence works. In particular, this code: -> -> 1..10 |> Enum.filter &(&1 <= 5) |> Enum.map &(&1 * 2) -> -> Actually translates to: -> -> 1..10 |> Enum.filter(&(&1 <= 5) |> Enum.map(&(&1 * 2))) -> -> Which is not what we want, since the function given to `Enum.filter/2` is the one passed as first argument to `Enum.map/2`. The solution is to use explicit parentheses: -> -> 1..10 |> Enum.filter(&(&1 <= 5)) |> Enum.map(&(&1 * 2)) -> - -The `read_line/1` implementation receives data from the socket using `:gen_tcp.recv/2` and `write_line/2` writes to the socket using `:gen_tcp.send/2`. - -This is pretty much all we need to implement our echo server. Let's give it a try! - -Start an iex session inside the `kv_server` application with `iex -S mix`. Inside IEx, run: - -```elixir -iex> KVServer.accept(4040) -``` - -The server is now running, and you will even notice the console is blocked. Let's use [a `telnet` client](http://en.wikipedia.org/wiki/Telnet) to access our server. There are clients available on most operating systems, and their command lines are generally similar: - - $ telnet 127.0.0.1 4040 - Trying 127.0.0.1... - Connected to localhost. - Escape character is '^]'. - hello - hello - is it me - is it me - you are looking for? - you are looking for? - -Type "hello", press enter, and you will get "hello" back. Excellent! - -My particular telnet client can be exited by typing `ctrl + ]`, typing `quit`, and pressing ``, but your client may require different steps. - -Once you exit the telnet client, you will likely see an error in the IEx session: - - ** (MatchError) no match of right hand side value: {:error, :closed} - (kv_server) lib/kv_server.ex:41: KVServer.read_line/1 - (kv_server) lib/kv_server.ex:33: KVServer.serve/1 - (kv_server) lib/kv_server.ex:27: KVServer.loop_acceptor/1 - -That's because we were expecting data from `:gen_tcp.recv/2` but the client closed the connection. We need to handle such cases better in future revisions of our server. - -For now there is a more important bug we need to fix: what happens if our TCP acceptor crashes? Since there is no supervision, the server dies and we won't be able to serve more requests, because it won't be restarted. That's why we must move our server inside a supervision tree. - -## 8.2 Tasks - -We have learned about agents, generic servers, and event managers. They are all meant to work with multiple messages or manage state. But what do we use when we only need to execute some task and that is it? - -[The Task module](/docs/stable/elixir/Task.html) provides this functionality exactly. For example, it has `start_link/3` function that receives a module, function and arguments, allowing us to run a given function as part of a supervision tree. - -Let's give it a try. Open up `lib/kv_server.ex`, and let's change the supervisor in the `start/2` function to the following: - -```elixir -def start(_type, _args) do - import Supervisor.Spec - - children = [ - worker(Task, [KVServer, :accept, [4040]]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) -end -``` - -With this change, we are saying that we want to run `KVServer.accept(4040)` as a worker. We are hardcoding the port for now, but we will discuss ways in which this could be changed later. - -Now that the server is part of the supervision tree, it should start automatically when we run the application. Type `mix run --no-halt` in the terminal, and once again use the `telnet` client to make sure that everything still works: - - $ telnet 127.0.0.1 4040 - Trying 127.0.0.1... - Connected to localhost. - Escape character is '^]'. - say you - say you - say me - say me - -Yes, it works! If you kill the client, causing the whole server to crash, you will see another one starts right away. However, does it *scale*? - -Try to connect two telnet clients at the same time. When you do so, you will notice that the second client doesn't echo: - - $ telnet 127.0.0.1 4040 - Trying 127.0.0.1... - Connected to localhost. - Escape character is '^]'. - hello - hello? - HELLOOOOOO? - -It doesn't seem to work at all. That's because we are serving requests in the same process that are accepting connections. When one client is connected, we can't accept another client. - -## 8.3 Task supervisor - -In order to make our server handle simultaneous connections, we need to have one process working as an acceptor that spawns other processes to serve requests. One solution would be to change: - -```elixir -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - serve(client) - loop_acceptor(socket) -end -``` - -to use `Task.start_link/1`, which is similar to `Task.start_link/3`, but it receives an anonymous function instead of module, function and arguments: - -```elixir -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - Task.start_link(fn -> serve(client) end) - loop_acceptor(socket) -end -``` - -But we've already made this mistake once. Do you remember? - -This is similar to the mistake we made when we called `KV.Bucket.start_link/0` from the registry. That meant a failure in any bucket would bring the whole registry down. - -The code above would have the same flaw: if we link the `serve(client)` task to the acceptor, a crash when serving a request would bring the acceptor, and consequently all other connections, down. - -We fixed the issue for the registry by using a simple one for one supervisor. We are going to use the same tactic here, except that this pattern is so common with tasks that tasks already come with a solution: a simple one for one supervisor with temporary workers that we can just use in our supervision tree! - -Let's change `start/2` once again, to add a supervisor to our tree: - -```elixir -def start(_type, _args) do - import Supervisor.Spec - - children = [ - supervisor(Task.Supervisor, [[name: KVServer.TaskSupervisor]]), - worker(Task, [KVServer, :accept, [4040]]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) -end -``` - -We simply start a [`Task.Supervisor`](/docs/stable/elixir/Task.Supervisor.html) process with name `KVServer.TaskSupervisor`. Remember, since the acceptor task depends on this supervisor, the supervisor must be started first. - -Now we just need to change `loop_acceptor/2` to use `Task.Supervisor` to serve each request: - -```elixir -defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - Task.Supervisor.start_child(KVServer.TaskSupervisor, fn -> serve(client) end) - loop_acceptor(socket) -end -``` - -Start a new server with `mix run --no-halt` and we can now open up many concurrent telnet clients. You will also notice that quitting a client does not bring the acceptor down. Excellent! - -Here is the full echo server implementation, in a single module: - -```elixir -defmodule KVServer do - use Application - - @doc false - def start(_type, _args) do - import Supervisor.Spec - - children = [ - supervisor(Task.Supervisor, [[name: KVServer.TaskSupervisor]]), - worker(Task, [KVServer, :accept, [4040]]) - ] - - opts = [strategy: :one_for_one, name: KVServer.Supervisor] - Supervisor.start_link(children, opts) - end - - @doc """ - Starts accepting connections on the given `port`. - """ - def accept(port) do - {:ok, socket} = :gen_tcp.listen(port, - [:binary, packet: :line, active: false]) - IO.puts "Accepting connections on port #{port}" - loop_acceptor(socket) - end - - defp loop_acceptor(socket) do - {:ok, client} = :gen_tcp.accept(socket) - Task.Supervisor.start_child(KVServer.TaskSupervisor, fn -> serve(client) end) - loop_acceptor(socket) - end - - defp serve(socket) do - socket - |> read_line() - |> write_line(socket) - - serve(socket) - end - - defp read_line(socket) do - {:ok, data} = :gen_tcp.recv(socket, 0) - data - end - - defp write_line(line, socket) do - :gen_tcp.send(socket, line) - end -end -``` - -Since we have changed the supervisor specification, we need to ask: is our supervision strategy is still correct? - -In this case, the answer is yes: if the acceptor crashes, there is no need to crash the existing connections. On the other hand, if the task supervisor crashes, there is no need to crash the acceptor too. This is a contrast to the registry, where we initially had to crash the supervisor every time the registry crashed, until we used ETS to persist state. However, tasks have no state and nothing will go stale if one of these processes dies. - -In the next chapter we will start parsing the client requests and sending responses, finishing our server. diff --git a/getting_started/mix_otp/9.markdown b/getting_started/mix_otp/9.markdown deleted file mode 100644 index 607f6ca4f..000000000 --- a/getting_started/mix_otp/9.markdown +++ /dev/null @@ -1,454 +0,0 @@ ---- -layout: getting_started -title: 9 Docs, tests and pipelines -guide: 9 ---- - -# {{ page.title }} - -
      - -In this chapter, we will implement the code that parses the commands we described in the first chapter: - -``` -CREATE shopping -OK - -PUT shopping milk 1 -OK - -PUT shopping eggs 3 -OK - -GET shopping milk -1 -OK - -DELETE shopping eggs -OK -``` - -After the parsing is done, we will update our server to dispatch the parsed commands to the `:kv` application we built previously. - -## 9.1 Doctests - -On the language homepage, we mention that Elixir makes documentation a first-class citizen in the language. We have explored this concept many times throughout this guide, be it via `mix help` or by typing `h Enum` or another module in an IEx console. - -In this section, we will implement the parse functionality using doctests, which allows us to write tests directly from our documentation. This helps us provide documentation with accurate code samples. - -Let's create our command parser at `lib/kv_server/command.ex` and start with the doctest: - -```elixir -defmodule KVServer.Command do - @doc ~S""" - Parses the given `line` into a command. - - ## Examples - - iex> KVServer.Command.parse "CREATE shopping\r\n" - {:ok, {:create, "shopping"}} - - """ - def parse(line) do - :not_implemented - end -end -``` - -Doctests are specified in by an indentation of four spaces followed by the `iex>` prompt in a documentation string. If a command spans multiple lines, you can use `...>`, as in IEx. The expected result should start at the next line after `iex>` or `...>` line(s) and is terminated either by a newline or a new `iex>` prefix. - -Also note that we started the documentation string using `@doc ~S"""`. The `~S` prevents the `\r\n` characters from being converted to a carriage return and line feed until they are evaluated in the test. - -To run our doctests, we'll create a file at `test/kv_server/command_test.exs` and call `doctest KVServer.Command` in the test case: - -```elixir -defmodule KVServer.CommandTest do - use ExUnit.Case, async: true - doctest KVServer.Command -end -``` - -Run the test suite and the doctest should fail: - - 1) test doc at KVServer.Command.parse/1 (1) (KVServer.CommandTest) - test/kv_server/command_test.exs:3 - Doctest failed - code: KVServer.Command.parse "CREATE shopping\r\n" === {:ok, {:create, "shopping"}} - lhs: :not_implemented - stacktrace: - lib/kv_server/command.ex:11: KVServer.Command (module) - -Excellent! - -Now it is just a matter of making the doctest pass. Let's implement the `parse/1` function: - -```elixir -def parse(line) do - case String.split(line) do - ["CREATE", bucket] -> {:ok, {:create, bucket}} - end -end -``` - -Our implementation simply splits the line on whitespace and then matches the command against a list. Using `String.split/1` means our commands will be whitespace-insensitive. Leading and trailing whitespace won't matter, nor will consecutive spaces between words. Let's add some new doctests to test this behaviour along with the other commands: - -```elixir -@doc ~S""" -Parses the given `line` into a command. - -## Examples - - iex> KVServer.Command.parse "CREATE shopping\r\n" - {:ok, {:create, "shopping"}} - - iex> KVServer.Command.parse "CREATE shopping \r\n" - {:ok, {:create, "shopping"}} - - iex> KVServer.Command.parse "PUT shopping milk 1\r\n" - {:ok, {:put, "shopping", "milk", "1"}} - - iex> KVServer.Command.parse "GET shopping milk\r\n" - {:ok, {:get, "shopping", "milk"}} - - iex> KVServer.Command.parse "DELETE shopping eggs\r\n" - {:ok, {:delete, "shopping", "eggs"}} - -Unknown commands or commands with the wrong number of -arguments return an error: - - iex> KVServer.Command.parse "UNKNOWN shopping eggs\r\n" - {:error, :unknown_command} - - iex> KVServer.Command.parse "GET shopping\r\n" - {:error, :unknown_command} - -""" -``` - -With doctests at hand, it is your turn to make tests pass! Once you're ready, you can compare your work with our solution below: - -```elixir -def parse(line) do - case String.split(line) do - ["CREATE", bucket] -> {:ok, {:create, bucket}} - ["GET", bucket, key] -> {:ok, {:get, bucket, key}} - ["PUT", bucket, key, value] -> {:ok, {:put, bucket, key, value}} - ["DELETE", bucket, key] -> {:ok, {:delete, bucket, key}} - _ -> {:error, :unknown_command} - end -end -``` - -Notice how we were able to elegantly parse the commands without adding a bunch of `if/else` clauses that check the command name and number of arguments! - -Finally, you may have observed that each doctest was considered to be a different test in our test case, as our test suite now reports a total of 7 tests. That is because ExUnit considers the following to define two different tests: - -```iex -iex> KVServer.Command.parse "UNKNOWN shopping eggs\r\n" -{:error, :unknown_command} - -iex> KVServer.Command.parse "GET shopping\r\n" -{:error, :unknown_command} -``` - -Without new lines, as seen below, ExUnit compiles it into a single test: - -```iex -iex> KVServer.Command.parse "UNKNOWN shopping eggs\r\n" -{:error, :unknown_command} -iex> KVServer.Command.parse "GET shopping\r\n" -{:error, :unknown_command} -``` - -You can read more about doctests in [the `ExUnit.DocTest` docs](/docs/stable/ex_unit/ExUnit.DocTest.html). - -## 9.2 Pipelines - -With our command parser in hand, we can finally start implementing the logic that runs the commands. Let's add a stub definition for this function for now: - -```elixir -defmodule KVServer.Command do - @doc """ - Runs the given command. - """ - def run(command) do - {:ok, "OK\r\n"} - end -end -``` - -Before we implement this function, let's change our server to start using our new `parse/1` and `run/1` functions. Remember, our `read_line/1` function was also crashing when the client closed the socket, so let's take the opportunity to fix it, too. Open up `lib/kv_server.ex` and replace the existing server definition: - -```elixir -defp serve(socket) do - socket - |> read_line() - |> write_line(socket) - - serve(socket) -end - -defp read_line(socket) do - {:ok, data} = :gen_tcp.recv(socket, 0) - data -end - -defp write_line(line, socket) do - :gen_tcp.send(socket, line) -end -``` - -with the following: - -```elixir -defp serve(socket) do - msg = - case read_line(socket) do - {:ok, data} -> - case KVServer.Command.parse(data) do - {:ok, command} -> - KVServer.Command.run(command) - {:error, _} = err -> - err - end - {:error, _} = err -> - err - end - - write_line(socket, msg) - serve(socket) -end - -defp read_line(socket) do - :gen_tcp.recv(socket, 0) -end - -defp write_line(socket, msg) do - :gen_tcp.send(socket, format_msg(msg)) -end - -defp format_msg({:ok, text}), do: text -defp format_msg({:error, :unknown_command}), do: "UNKNOWN COMMAND\r\n" -defp format_msg({:error, _}), do: "ERROR\r\n" -``` - -If we start our server, we can now send commands to it. For now we will get two different responses: "OK" when the command is known and "UNKNOWN COMMAND" otherwise: - - $ telnet 127.0.0.1 4040 - Trying 127.0.0.1... - Connected to localhost. - Escape character is '^]'. - CREATE shopping - OK - HELLO - UNKNOWN COMMAND - -This means our implementation is going in the correct direction, but it doesn't look very elegant, does it? - -The previous implementation used pipes which made the logic straight-forward to understand: - -```elixir -read_line(socket) |> KVServer.Command.parse |> KVServer.Command.run() -``` - -Since we may have failures along the way, we need our pipeline logic to match error outputs and abort if they occur. Wouldn't it be great if instead we could say: "pipe these functions while the response is `:ok`" or "pipe these functions while the response matches the `{:ok, _}` tuple"? - -Thankfully, there is a project called [elixir-pipes](https://github.com/batate/elixir-pipes) that provides exactly this functionality! Let's give it a try. - -Open up your `apps/kv_server/mix.exs` file and change both `application/0` and `deps/0` functions to the following: - -```elixir -def application do - [applications: [:logger, :pipe, :kv], - mod: {KVServer, []}] -end - -defp deps do - [{:kv, in_umbrella: true}, - {:pipe, github: "batate/elixir-pipes"}] -end -``` - -Run `mix deps.get` to get the dependency, and rewrite the `serve/1` function to use the `pipe_matching/3` functionality now available to us: - -```elixir -defp serve(socket) do - import Pipe - - msg = - pipe_matching x, {:ok, x}, - read_line(socket) - |> KVServer.Command.parse() - |> KVServer.Command.run() - - write_line(socket, msg) - serve(socket) -end -``` - -With `pipe_matching/3` we can ask Elixir to pipe the value `x` from each step if it matches `{:ok, x}`. We do so by basically converting each expression given to `case/2` as a step in the pipeline. As soon as any of the steps return something that does not match `{:ok, x}`, the pipeline aborts, and returns the non-matching value. - -Excellent! Feel free to read the [elixir-pipes](https://github.com/batate/elixir-pipes) project documentation to learn about other options for expressing pipelines. Let's continue moving forward with our server implementation. - -## 9.3 Running commands - -The last step is to implement `KVServer.Command.run/1`, to run the parsed commands against the `:kv` application. Its implementation is shown below: - -```elixir -@doc """ -Runs the given command. -""" -def run(command) - -def run({:create, bucket}) do - KV.Registry.create(KV.Registry, bucket) - {:ok, "OK\r\n"} -end - -def run({:get, bucket, key}) do - lookup bucket, fn pid -> - value = KV.Bucket.get(pid, key) - {:ok, "#{value}\r\nOK\r\n"} - end -end - -def run({:put, bucket, key, value}) do - lookup bucket, fn pid -> - KV.Bucket.put(pid, key, value) - {:ok, "OK\r\n"} - end -end - -def run({:delete, bucket, key}) do - lookup bucket, fn pid -> - KV.Bucket.delete(pid, key) - {:ok, "OK\r\n"} - end -end - -defp lookup(bucket, callback) do - case KV.Registry.lookup(KV.Registry, bucket) do - {:ok, pid} -> callback.(pid) - :error -> {:error, :not_found} - end -end -``` - -The implementation is straightforward: we just dispatch to the `KV.Registry` server that we registered during the `:kv` application startup. - -Note that we have also defined a private function named `lookup/2` to help with the common functionality of looking up a bucket and returning its `pid` if it exists, `{:error, :not_found}` otherwise. - -By the way, since we are now returning `{:error, :not_found}`, we should amend the `format_msg/1` function in `KV.Server` to nicely show not found messages too: - -```elixir -defp format_msg({:ok, text}), do: text -defp format_msg({:error, :unknown_command}), do: "UNKNOWN COMMAND\r\n" -defp format_msg({:error, :not_found}), do: "NOT FOUND\r\n" -defp format_msg({:error, _}), do: "ERROR\r\n" -``` - -And our server functionality is almost complete! We just need to add tests. This time, we have left tests for last because there are some important considerations to be made. - -`KVServer.Command.run/1`'s implementation is sending commands directly to the server named `KV.Registry`, which is registered by the `:kv` application. This means this server is global and if we have two tests sending messages to it at the same time, our tests will conflict with each other (and likely fail). We need to decide between having unit tests that are isolated and can run asynchronously, or writing integration tests that work on top of the global state, but exercise our application's full stack as it is meant to be exercised in production. - -So far we have been chosing the unit test approach. For example, in order to make `KVServer.Command.run/1` testable as a unit we would need to change its implementation to not send commands directly to the `KV.Registry` process but instead pass a server as argument. This means we would need to change `run`'s signature to `def run(command, pid)` and the implementation for the `:create` command would look like: - -```elixir -def run({:create, bucket}, pid) do - KV.Registry.create(pid, bucket) - {:ok, "OK\r\n"} -end -``` - -Then in `KVServer.Command`'s test case, we would need to start an instance of the `KV.Registry`, similar to what we've done in `apps/kv/test/kv/registry_test.exs`, and pass it as an argument to `run/2`. - -This has been the approach we have taken so far in our tests, and it has some benefits: - -1. Our implementation is not coupled to any particular server name -2. We can keep our tests running asynchronously, because there is no shared state - -However, it comes with the downside that our APIs become increasingly large in order to accommodate all external parameters. - -The alternative is to continue relying on the global server names and run tests against the global data, ensuring we clean up the data in between the tests. In this case, since the test would exercise the whole stack, from the TCP server, to the command parsing and running, to the registry and finally reaching the bucket, it becomes an integration test. - -The downside of integration tests is that they can be much slower than unit tests, and as such they must be used more sparingly. For example, we should not use integration tests to test an edge case in our command parsing implementation. - -Since we have used unit tests so far, this time we will take the other road and write an integration test. The integration test will have a TCP client that sends commands to our server and we will assert that we are getting the desired responses. - -Let's implement our integration test in `test/kv_server_test.exs` as shown below: - -```elixir -defmodule KVServerTest do - use ExUnit.Case - - setup do - :application.stop(:kv) - :ok = :application.start(:kv) - end - - setup do - opts = [:binary, packet: :line, active: false] - {:ok, socket} = :gen_tcp.connect('localhost', 4040, opts) - {:ok, socket: socket} - end - - test "server interaction", %{socket: socket} do - assert send_and_recv(socket, "UNKNOWN shopping\r\n") == - "UNKNOWN COMMAND\r\n" - - assert send_and_recv(socket, "GET shopping eggs\r\n") == - "NOT FOUND\r\n" - - assert send_and_recv(socket, "CREATE shopping\r\n") == - "OK\r\n" - - assert send_and_recv(socket, "PUT shopping eggs 3\r\n") == - "OK\r\n" - - # GET returns two lines - assert send_and_recv(socket, "GET shopping eggs\r\n") == "3\r\n" - assert send_and_recv(socket, "") == "OK\r\n" - - assert send_and_recv(socket, "DELETE shopping eggs\r\n") == - "OK\r\n" - - # GET returns two lines - assert send_and_recv(socket, "GET shopping eggs\r\n") == "\r\n" - assert send_and_recv(socket, "") == "OK\r\n" - end - - defp send_and_recv(socket, command) do - :ok = :gen_tcp.send(socket, command) - {:ok, data} = :gen_tcp.recv(socket, 0, 1000) - data - end -end -``` - -Our integration test checks all server interaction, including unknown commands and not found errors. It is worth noting that, as with ETS tables and linked processes, there is no need to close the socket. Once the test process exits, the socket is automatically closed. - -This time, since our test relies on global data, we have not given `async: true` to `use ExUnit.Case`. Furthermore, in order to guarantee our test is always in a clean state, we stop and start the `:kv` application before each test. In fact, stopping the `:kv` application even prints a warning on the terminal: - -``` -18:12:10.698 [info] Application kv exited with reason :stopped -``` - -If desired, we can avoid printing this warning by turning the error_logger off and on in the test setup: - -```elixir -setup do - Logger.remove_backend(:console) - Application.stop(:kv) - :ok = Application.start(:kv) - Logger.add_backend(:console, flush: true) - :ok -end -``` - -With this simple integration test, we start to see why integration tests may be slow. Not only can this particular test not be run asynchronously, it also requires the expensive setup of stopping and starting the `:kv` application. - -At the end of the day, it is up to you and your team to figure out the best testing strategy for your applications. You need to balance code quality, confidence, and test suite runtime. For example, we may start with testing the server only with integration tests, but if the server continues to grow in future releases, or it becomes a part of the application with frequent bugs, it is important to consider breaking it apart and writing more intensive unit tests that don't have the weight of an integration test. - -I personally err on the side of unit tests, and have integration tests only as smoke tests to guarantee the basic skeleton of the system works. - -In the next chapter we will finally make our system distributed by adding a bucket routing mechanism. We'll also learn about application configuration. diff --git a/images/cases/bg/change-graph.png b/images/cases/bg/change-graph.png new file mode 100644 index 000000000..41e70fc34 Binary files /dev/null and b/images/cases/bg/change-graph.png differ diff --git a/images/cases/bg/change.png b/images/cases/bg/change.png new file mode 100644 index 000000000..7bf0d37a0 Binary files /dev/null and b/images/cases/bg/change.png differ diff --git a/images/cases/bg/cyanview-1.jpg b/images/cases/bg/cyanview-1.jpg new file mode 100644 index 000000000..2e0b9ffcf Binary files /dev/null and b/images/cases/bg/cyanview-1.jpg differ diff --git a/images/cases/bg/cyanview-2.jpg b/images/cases/bg/cyanview-2.jpg new file mode 100644 index 000000000..a6e7796e6 Binary files /dev/null and b/images/cases/bg/cyanview-2.jpg differ diff --git a/images/cases/bg/cyanview-3.jpg b/images/cases/bg/cyanview-3.jpg new file mode 100644 index 000000000..fbe3515f3 Binary files /dev/null and b/images/cases/bg/cyanview-3.jpg differ diff --git a/images/cases/bg/cyanview-4.jpg b/images/cases/bg/cyanview-4.jpg new file mode 100644 index 000000000..c14baed64 Binary files /dev/null and b/images/cases/bg/cyanview-4.jpg differ diff --git a/images/cases/bg/discord.jpg b/images/cases/bg/discord.jpg new file mode 100755 index 000000000..06e482dd7 Binary files /dev/null and b/images/cases/bg/discord.jpg differ diff --git a/images/cases/bg/duffel.png b/images/cases/bg/duffel.png new file mode 100644 index 000000000..d06a6e9a6 Binary files /dev/null and b/images/cases/bg/duffel.png differ diff --git a/images/cases/bg/farmbot.jpg b/images/cases/bg/farmbot.jpg new file mode 100644 index 000000000..da5dda3e3 Binary files /dev/null and b/images/cases/bg/farmbot.jpg differ diff --git a/images/cases/bg/heroku.png b/images/cases/bg/heroku.png new file mode 100644 index 000000000..04454907f Binary files /dev/null and b/images/cases/bg/heroku.png differ diff --git a/images/cases/bg/met-france.png b/images/cases/bg/met-france.png new file mode 100644 index 000000000..c79da2fb1 Binary files /dev/null and b/images/cases/bg/met-france.png differ diff --git a/images/cases/bg/mozilla-hubs.jpg b/images/cases/bg/mozilla-hubs.jpg new file mode 100644 index 000000000..5b78639d9 Binary files /dev/null and b/images/cases/bg/mozilla-hubs.jpg differ diff --git a/images/cases/bg/pepsico.jpg b/images/cases/bg/pepsico.jpg new file mode 100644 index 000000000..e9f6be8de Binary files /dev/null and b/images/cases/bg/pepsico.jpg differ diff --git a/images/cases/bg/remote.png b/images/cases/bg/remote.png new file mode 100644 index 000000000..a4285ad4e Binary files /dev/null and b/images/cases/bg/remote.png differ diff --git a/images/cases/bg/slab.png b/images/cases/bg/slab.png new file mode 100644 index 000000000..6f5a0e3dc Binary files /dev/null and b/images/cases/bg/slab.png differ diff --git a/images/cases/bg/sparkmeter-new-architecture.png b/images/cases/bg/sparkmeter-new-architecture.png new file mode 100644 index 000000000..9ed82cabe Binary files /dev/null and b/images/cases/bg/sparkmeter-new-architecture.png differ diff --git a/images/cases/bg/sparkmeter-old-architecture.png b/images/cases/bg/sparkmeter-old-architecture.png new file mode 100644 index 000000000..a5acca6ba Binary files /dev/null and b/images/cases/bg/sparkmeter-old-architecture.png differ diff --git a/images/cases/bg/v7.png b/images/cases/bg/v7.png new file mode 100644 index 000000000..59391bf33 Binary files /dev/null and b/images/cases/bg/v7.png differ diff --git a/images/cases/bg/x-plane.jpg b/images/cases/bg/x-plane.jpg new file mode 100644 index 000000000..10da67180 Binary files /dev/null and b/images/cases/bg/x-plane.jpg differ diff --git a/images/cases/logos/change.png b/images/cases/logos/change.png new file mode 100644 index 000000000..9d002c59a Binary files /dev/null and b/images/cases/logos/change.png differ diff --git a/images/cases/logos/community.png b/images/cases/logos/community.png new file mode 100644 index 000000000..11e6cff7b Binary files /dev/null and b/images/cases/logos/community.png differ diff --git a/images/cases/logos/cyanview.png b/images/cases/logos/cyanview.png new file mode 100644 index 000000000..4fac3bf69 Binary files /dev/null and b/images/cases/logos/cyanview.png differ diff --git a/images/cases/logos/default-image.png b/images/cases/logos/default-image.png new file mode 100644 index 000000000..edcfde32a Binary files /dev/null and b/images/cases/logos/default-image.png differ diff --git a/images/cases/logos/discord.png b/images/cases/logos/discord.png new file mode 100644 index 000000000..aa1b5e2e0 Binary files /dev/null and b/images/cases/logos/discord.png differ diff --git a/images/cases/logos/duffel.png b/images/cases/logos/duffel.png new file mode 100644 index 000000000..a2c197a8b Binary files /dev/null and b/images/cases/logos/duffel.png differ diff --git a/images/cases/logos/farmbot.png b/images/cases/logos/farmbot.png new file mode 100644 index 000000000..d8d5a914b Binary files /dev/null and b/images/cases/logos/farmbot.png differ diff --git a/images/cases/logos/heroku.png b/images/cases/logos/heroku.png new file mode 100644 index 000000000..78f52eda4 Binary files /dev/null and b/images/cases/logos/heroku.png differ diff --git a/images/cases/logos/met-france.svg b/images/cases/logos/met-france.svg new file mode 100644 index 000000000..09ff2ae0a --- /dev/null +++ b/images/cases/logos/met-france.svg @@ -0,0 +1,3 @@ + + +Accueil - Ministère de la Transition écologiqueMinistère de la Transition écologique diff --git a/images/cases/logos/mozilla-hubs.png b/images/cases/logos/mozilla-hubs.png new file mode 100644 index 000000000..f1b7779c3 Binary files /dev/null and b/images/cases/logos/mozilla-hubs.png differ diff --git a/images/cases/logos/pepsico.png b/images/cases/logos/pepsico.png new file mode 100644 index 000000000..308de0c1f Binary files /dev/null and b/images/cases/logos/pepsico.png differ diff --git a/images/cases/logos/remote.png b/images/cases/logos/remote.png new file mode 100644 index 000000000..6e8db442f Binary files /dev/null and b/images/cases/logos/remote.png differ diff --git a/images/cases/logos/slab.png b/images/cases/logos/slab.png new file mode 100644 index 000000000..9dcf90f89 Binary files /dev/null and b/images/cases/logos/slab.png differ diff --git a/images/cases/logos/sparkmeter.png b/images/cases/logos/sparkmeter.png new file mode 100644 index 000000000..500414dcb Binary files /dev/null and b/images/cases/logos/sparkmeter.png differ diff --git a/images/cases/logos/v7.png b/images/cases/logos/v7.png new file mode 100644 index 000000000..9acd9dfb7 Binary files /dev/null and b/images/cases/logos/v7.png differ diff --git a/images/cases/logos/veeps.svg b/images/cases/logos/veeps.svg new file mode 100755 index 000000000..8e9ce4fa5 --- /dev/null +++ b/images/cases/logos/veeps.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/images/cases/logos/x-plane.png b/images/cases/logos/x-plane.png new file mode 100644 index 000000000..40da79cc0 Binary files /dev/null and b/images/cases/logos/x-plane.png differ diff --git a/images/contents/debugger-elixir.gif b/images/contents/debugger-elixir.gif new file mode 100644 index 000000000..0d99d2a71 Binary files /dev/null and b/images/contents/debugger-elixir.gif differ diff --git a/images/contents/deps-tree-phoenix.svg b/images/contents/deps-tree-phoenix.svg new file mode 100644 index 000000000..23f1146fb --- /dev/null +++ b/images/contents/deps-tree-phoenix.svg @@ -0,0 +1 @@ +phoenixpoison~> 1.5 or ~> 2.0phoenix_pubsub~> 1.0.0-rccowboy~> 1.0plug~> 1.1cowlib~> 1.0.0ranch~> 1.0~> 1.0 \ No newline at end of file diff --git a/images/contents/exunit-ansi.png b/images/contents/exunit-ansi.png index d6ce6aab3..eef145e9e 100644 Binary files a/images/contents/exunit-ansi.png and b/images/contents/exunit-ansi.png differ diff --git a/images/contents/exunit-bare-assertion-diff.png b/images/contents/exunit-bare-assertion-diff.png new file mode 100644 index 000000000..57a7bf7e7 Binary files /dev/null and b/images/contents/exunit-bare-assertion-diff.png differ diff --git a/images/contents/exunit-diff.png b/images/contents/exunit-diff.png new file mode 100644 index 000000000..4bb568d91 Binary files /dev/null and b/images/contents/exunit-diff.png differ diff --git a/images/contents/fast-fail.png b/images/contents/fast-fail.png index 10f53e353..4f2938511 100644 Binary files a/images/contents/fast-fail.png and b/images/contents/fast-fail.png differ diff --git a/images/contents/gem.jpeg b/images/contents/gem.jpeg new file mode 100644 index 000000000..220a094c6 Binary files /dev/null and b/images/contents/gem.jpeg differ diff --git a/images/contents/hash-dict-fetch.png b/images/contents/hash-dict-fetch.png index 31abaf84d..6af8ad47d 100644 Binary files a/images/contents/hash-dict-fetch.png and b/images/contents/hash-dict-fetch.png differ diff --git a/images/contents/hash-dict-update.png b/images/contents/hash-dict-update.png index 8713606ab..d5a00d498 100644 Binary files a/images/contents/hash-dict-update.png and b/images/contents/hash-dict-update.png differ diff --git a/images/contents/home-code.png b/images/contents/home-code.png deleted file mode 100644 index 482aaa084..000000000 Binary files a/images/contents/home-code.png and /dev/null differ diff --git a/images/contents/iex-auto-reload.mp4 b/images/contents/iex-auto-reload.mp4 new file mode 100644 index 000000000..d21ad3f8b Binary files /dev/null and b/images/contents/iex-auto-reload.mp4 differ diff --git a/images/contents/iex-coloring.png b/images/contents/iex-coloring.png new file mode 100644 index 000000000..7e3e4a67a Binary files /dev/null and b/images/contents/iex-coloring.png differ diff --git a/images/contents/iex-metadata.png b/images/contents/iex-metadata.png new file mode 100644 index 000000000..709922955 Binary files /dev/null and b/images/contents/iex-metadata.png differ diff --git a/images/contents/iex-pry.png b/images/contents/iex-pry.png index 567375606..6c7ae953b 100644 Binary files a/images/contents/iex-pry.png and b/images/contents/iex-pry.png differ diff --git a/images/contents/kv-observer.png b/images/contents/kv-observer.png new file mode 100644 index 000000000..7527d7e5c Binary files /dev/null and b/images/contents/kv-observer.png differ diff --git a/images/contents/livebook-boot-1.15.png b/images/contents/livebook-boot-1.15.png new file mode 100644 index 000000000..f603151fc Binary files /dev/null and b/images/contents/livebook-boot-1.15.png differ diff --git a/images/contents/livebook-compile-1.15.png b/images/contents/livebook-compile-1.15.png new file mode 100644 index 000000000..58f41504f Binary files /dev/null and b/images/contents/livebook-compile-1.15.png differ diff --git a/images/contents/string-help.png b/images/contents/string-help.png index a0938761e..3831a5362 100644 Binary files a/images/contents/string-help.png and b/images/contents/string-help.png differ diff --git a/images/contents/type-warning-case.png b/images/contents/type-warning-case.png new file mode 100644 index 000000000..118dad1f1 Binary files /dev/null and b/images/contents/type-warning-case.png differ diff --git a/images/contents/type-warning-function-clause.png b/images/contents/type-warning-function-clause.png new file mode 100644 index 000000000..4c2fa76dd Binary files /dev/null and b/images/contents/type-warning-function-clause.png differ diff --git a/images/contents/type-warning-in-editor.png b/images/contents/type-warning-in-editor.png new file mode 100644 index 000000000..38293a0c8 Binary files /dev/null and b/images/contents/type-warning-in-editor.png differ diff --git a/images/contents/type-warning-on-date-comparison.png b/images/contents/type-warning-on-date-comparison.png new file mode 100644 index 000000000..349372cd6 Binary files /dev/null and b/images/contents/type-warning-on-date-comparison.png differ diff --git a/images/contents/type-warning-on-struct-field.png b/images/contents/type-warning-on-struct-field.png new file mode 100644 index 000000000..1d003bf3c Binary files /dev/null and b/images/contents/type-warning-on-struct-field.png differ diff --git a/images/contents/type-warning-private.png b/images/contents/type-warning-private.png new file mode 100644 index 000000000..da05ab1cb Binary files /dev/null and b/images/contents/type-warning-private.png differ diff --git a/images/learning/adopting-elixir.jpg b/images/learning/adopting-elixir.jpg new file mode 100644 index 000000000..77ec6a2ea Binary files /dev/null and b/images/learning/adopting-elixir.jpg differ diff --git a/images/learning/alchemist-camp.png b/images/learning/alchemist-camp.png new file mode 100644 index 000000000..f6704071c Binary files /dev/null and b/images/learning/alchemist-camp.png differ diff --git a/images/learning/concurrent-data-processing.jpg b/images/learning/concurrent-data-processing.jpg new file mode 100644 index 000000000..5c9bd0a95 Binary files /dev/null and b/images/learning/concurrent-data-processing.jpg differ diff --git a/images/learning/designing-elixir-systems.jpg b/images/learning/designing-elixir-systems.jpg new file mode 100644 index 000000000..68413bd94 Binary files /dev/null and b/images/learning/designing-elixir-systems.jpg differ diff --git a/images/learning/educative-io-metaprogramming.png b/images/learning/educative-io-metaprogramming.png new file mode 100644 index 000000000..bf143b471 Binary files /dev/null and b/images/learning/educative-io-metaprogramming.png differ diff --git a/images/learning/elixir-cookbook.jpg b/images/learning/elixir-cookbook.jpg new file mode 100644 index 000000000..ecd8fe352 Binary files /dev/null and b/images/learning/elixir-cookbook.jpg differ diff --git a/images/learning/elixir-in-action.jpg b/images/learning/elixir-in-action.jpg new file mode 100644 index 000000000..932bce213 Binary files /dev/null and b/images/learning/elixir-in-action.jpg differ diff --git a/images/learning/elixir-koans-logo.png b/images/learning/elixir-koans-logo.png new file mode 100644 index 000000000..fe9de9f24 Binary files /dev/null and b/images/learning/elixir-koans-logo.png differ diff --git a/images/learning/elixir-school.jpg b/images/learning/elixir-school.jpg new file mode 100644 index 000000000..61dc24175 Binary files /dev/null and b/images/learning/elixir-school.jpg differ diff --git a/images/learning/elixir-sips.png b/images/learning/elixir-sips.png new file mode 100644 index 000000000..3c6aea8f0 Binary files /dev/null and b/images/learning/elixir-sips.png differ diff --git a/images/learning/elixir-streams-logo.png b/images/learning/elixir-streams-logo.png new file mode 100644 index 000000000..48e4434d3 Binary files /dev/null and b/images/learning/elixir-streams-logo.png differ diff --git a/images/learning/elixircasts.png b/images/learning/elixircasts.png new file mode 100644 index 000000000..b0052df5b Binary files /dev/null and b/images/learning/elixircasts.png differ diff --git a/images/learning/erlang-in-anger.png b/images/learning/erlang-in-anger.png new file mode 100644 index 000000000..9ba80a420 Binary files /dev/null and b/images/learning/erlang-in-anger.png differ diff --git a/images/learning/etude-for-elixir.jpg b/images/learning/etude-for-elixir.jpg new file mode 100644 index 000000000..549fafad4 Binary files /dev/null and b/images/learning/etude-for-elixir.jpg differ diff --git a/images/learning/flashcards_army.jpg b/images/learning/flashcards_army.jpg new file mode 100644 index 000000000..f40790ea6 Binary files /dev/null and b/images/learning/flashcards_army.jpg differ diff --git a/images/learning/groxio-elixir.png b/images/learning/groxio-elixir.png new file mode 100644 index 000000000..08949d5d3 Binary files /dev/null and b/images/learning/groxio-elixir.png differ diff --git a/images/learning/groxio-otp.png b/images/learning/groxio-otp.png new file mode 100644 index 000000000..bf919c08a Binary files /dev/null and b/images/learning/groxio-otp.png differ diff --git a/images/learning/introducing-elixir.jpg b/images/learning/introducing-elixir.jpg new file mode 100644 index 000000000..751ef4219 Binary files /dev/null and b/images/learning/introducing-elixir.jpg differ diff --git a/images/learning/joy-of-elixir.jpg b/images/learning/joy-of-elixir.jpg new file mode 100644 index 000000000..c197a1060 Binary files /dev/null and b/images/learning/joy-of-elixir.jpg differ diff --git a/images/learning/learn-elixir-dev.png b/images/learning/learn-elixir-dev.png new file mode 100644 index 000000000..63793ba8e Binary files /dev/null and b/images/learning/learn-elixir-dev.png differ diff --git a/images/learning/learn-elixir-tv.png b/images/learning/learn-elixir-tv.png new file mode 100644 index 000000000..3a134669b Binary files /dev/null and b/images/learning/learn-elixir-tv.png differ diff --git a/images/learning/learn-functional-programming-with-elixir.jpg b/images/learning/learn-functional-programming-with-elixir.jpg new file mode 100644 index 000000000..0251fd91b Binary files /dev/null and b/images/learning/learn-functional-programming-with-elixir.jpg differ diff --git a/images/learning/metaprogramming-elixir.jpg b/images/learning/metaprogramming-elixir.jpg new file mode 100644 index 000000000..44610ed70 Binary files /dev/null and b/images/learning/metaprogramming-elixir.jpg differ diff --git a/images/learning/mini-docu.png b/images/learning/mini-docu.png new file mode 100644 index 000000000..3a18117f8 Binary files /dev/null and b/images/learning/mini-docu.png differ diff --git a/images/learning/pragmaticstudio-elixir.png b/images/learning/pragmaticstudio-elixir.png new file mode 100644 index 000000000..0c8ab4100 Binary files /dev/null and b/images/learning/pragmaticstudio-elixir.png differ diff --git a/images/learning/programming-elixir-1-6.jpg b/images/learning/programming-elixir-1-6.jpg new file mode 100644 index 000000000..b61274161 Binary files /dev/null and b/images/learning/programming-elixir-1-6.jpg differ diff --git a/images/learning/running-in-production-logo.jpg b/images/learning/running-in-production-logo.jpg new file mode 100644 index 000000000..93ba4578a Binary files /dev/null and b/images/learning/running-in-production-logo.jpg differ diff --git a/images/learning/take-off-with-elixir.jpg b/images/learning/take-off-with-elixir.jpg new file mode 100644 index 000000000..c6b4ca144 Binary files /dev/null and b/images/learning/take-off-with-elixir.jpg differ diff --git a/images/learning/techschool.png b/images/learning/techschool.png new file mode 100644 index 000000000..3305082ee Binary files /dev/null and b/images/learning/techschool.png differ diff --git a/images/learning/the-little-elixir-and-otp-guidebook-meap.jpg b/images/learning/the-little-elixir-and-otp-guidebook-meap.jpg new file mode 100644 index 000000000..7ab2cf67d Binary files /dev/null and b/images/learning/the-little-elixir-and-otp-guidebook-meap.jpg differ diff --git a/images/learning/thinking-elixir-pattern-matching-course-tile-190px.png b/images/learning/thinking-elixir-pattern-matching-course-tile-190px.png new file mode 100644 index 000000000..9ac20f687 Binary files /dev/null and b/images/learning/thinking-elixir-pattern-matching-course-tile-190px.png differ diff --git a/images/learning/toy-robot.png b/images/learning/toy-robot.png new file mode 100644 index 000000000..bfbc6a932 Binary files /dev/null and b/images/learning/toy-robot.png differ diff --git a/images/logo/eef.png b/images/logo/eef.png new file mode 100644 index 000000000..0ffba654e Binary files /dev/null and b/images/logo/eef.png differ diff --git a/images/logo/logo-dark.png b/images/logo/logo-dark.png new file mode 100644 index 000000000..8beb02d62 Binary files /dev/null and b/images/logo/logo-dark.png differ diff --git a/images/logo/logo.png b/images/logo/logo.png old mode 100644 new mode 100755 index 9cc895cfe..9c13fb5a1 Binary files a/images/logo/logo.png and b/images/logo/logo.png differ diff --git a/images/logo/plataformatec.png b/images/logo/plataformatec.png deleted file mode 100644 index 06bc56cb8..000000000 Binary files a/images/logo/plataformatec.png and /dev/null differ diff --git a/images/social/RSSButton.png b/images/social/RSSButton.png deleted file mode 100644 index 26170ec89..000000000 Binary files a/images/social/RSSButton.png and /dev/null differ diff --git a/images/social/elixir-og-card.jpg b/images/social/elixir-og-card.jpg new file mode 100644 index 000000000..64cd0da91 Binary files /dev/null and b/images/social/elixir-og-card.jpg differ diff --git a/images/social/rss-feed-icon.svg b/images/social/rss-feed-icon.svg new file mode 100644 index 000000000..0435819b1 --- /dev/null +++ b/images/social/rss-feed-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/index.html b/index.html index 1a503cee9..66925b0e5 100644 --- a/index.html +++ b/index.html @@ -1,18 +1,48 @@ --- section: home layout: default +image: /images/social/elixir-og-card.jpg ---
      -
      - Elixir Sample - +
      -

      Elixir is a dynamic, functional language designed for building scalable and maintainable applications.

      +
      Elixir is a dynamic, functional language for building scalable and maintainable applications.
      -

      Elixir leverages the Erlang VM, known for running low-latency, distributed and fault-tolerant systems, while also being successfully used in web development and the embedded software domain.

      +

      Elixir runs on the Erlang VM, known for creating low-latency, distributed, and fault-tolerant systems. These capabilities and Elixir tooling allow developers to be productive in several domains, such as web development, embedded software, machine learning, data pipelines, and multimedia processing, across a wide range of industries.

      -

      To learn more about Elixir, check our getting started guide. Or keep reading to get an overview of the platform, language and tools.

      +

      Here is a peek:

      + +{% highlight elixir %} +iex> "Elixir" |> String.graphemes() |> Enum.frequencies() +%{"E" => 1, "i" => 2, "l" => 1, "r" => 1, "x" => 1} +{% endhighlight %} + +

      Check our Getting Started guide and our Learning page to begin your journey with Elixir. Or keep scrolling for an overview of the platform, language, and tools. +

      +
      + +
      +
      +

      Companies using Elixir in production

      + See more cases → +
      + +
      @@ -23,26 +53,25 @@

      Platform features

      Scalability

      -

      All Elixir code runs inside lightweight threads of execution (called processes) that are isolated and exchange information via messages:

      {% highlight elixir %} -parent = self() +current_process = self() -# Spawns an Elixir process (not an operating system one!) +# Spawn an Elixir process (not an operating system one!) spawn_link(fn -> - send parent, {:msg, "hello world"} + send(current_process, {:msg, "hello world"}) end) # Block until the message is received receive do - {:msg, contents} -> IO.puts contents + {:msg, contents} -> IO.puts(contents) end {% endhighlight %} -

      Due do their lightweight nature, it is not uncommon to have hundreds of thousands of processes running concurrently in the same machine. Isolation allows processes to be garbage collected independently, reducing system-wide pauses, and using all machine resources as efficiently as possible (vertical scaling).

      +

      Due to their lightweight nature, you can run hundreds of thousands of processes concurrently in the same machine, using all machine resources efficiently (vertical scaling). Processes may also communicate with other processes running on different machines to coordinate work across multiple nodes (horizontal scaling).

      -

      Processes are also able to communicate with other processes running on different machines in the same network. This provides the foundation for distribution, allowing developers to coordinate work across multiple nodes (horizontal scaling).

      +

      Together with projects such as Numerical Elixir, Elixir scales across cores, clusters, and GPUs.

      @@ -50,20 +79,20 @@

      Scalability

      Fault-tolerance

      -

      The unavoidable truth about software running in production is that things will go wrong. Even more when we take network, file systems and other third-party resources into account.

      +

      The unavoidable truth about software in production is that things will go wrong. Even more when we take network, file systems, and other third-party resources into account.

      -

      To cope with failures, Elixir provides supervisors which describe how to restart parts of your system when things go awry, going back to a known initial state that is guaranteed to work:

      +

      To react to failures, Elixir supervisors describe how to restart parts of your system when things go awry, going back to a known initial state that is guaranteed to work:

      {% highlight elixir %} -import Supervisor.Spec - children = [ - supervisor(TCP.Pool, []), - worker(TCP.Acceptor, [4040]) + TCP.Pool, + {TCP.Acceptor, port: 4040} ] Supervisor.start_link(children, strategy: :one_for_one) {% endhighlight %} + +

      The combination of fault-tolerance and message passing makes Elixir an excellent choice for event-driven systems and robust architectures. Frameworks, such as Nerves, build on this foundation to enable productive development of reliable embedded/IoT systems.

      @@ -75,25 +104,18 @@

      Language features

      Functional programming

      -

      Functional programming promotes a coding style that helps developers write code that is short, fast, and maintainable. For example, pattern matching allows developers to easily destructure data and access its contents:

      - -{% highlight elixir %} -%User{name: name, age: age} = User.get("John Doe") -name #=> "John Doe" -{% endhighlight %} - -

      When mixed with guards, pattern matching allows us to elegantly match and assert specific conditions for some code to execute:

      +

      Functional programming promotes a coding style that helps developers write code that is short, concise, and maintainable. For example, pattern matching allows us to elegantly match and assert specific conditions for some code to execute:

      {% highlight elixir %} -def serve_drinks(%User{age: age}) when age >= 21 do - # Code that serves drinks! +def drive(%User{age: age}) when age >= 16 do + # Code that drives a car end -serve_drinks User.get("John Doe") -#=> Fails if the user is under 21 +drive(User.get("John Doe")) +#=> Fails if the user is under 16 {% endhighlight %} -

      Elixir relies heavily on those features to ensure your software is working under the expected constraints. And when it is not, don't worry, supervisors got your back!

      +

      Elixir relies on those features to ensure your software is working under the expected constraints. And when it is not, don't worry, supervisors have your back!

      @@ -101,9 +123,9 @@

      Functional programming

      Extensibility and DSLs

      -

      Elixir has been designed to be extensible, letting developers naturally extend the language to particular domains, in order to increase their productivity.

      +

      Elixir has been designed to be extensible, allowing developers to naturally extend the language to particular domains, in order to increase their productivity.

      -

      As an example, let's write a simple test case using Elixir's test framework called ExUnit:

      +

      As an example, let's write a simple test case using Elixir's test framework called ExUnit:

      {% highlight elixir %} defmodule MathTest do @@ -115,7 +137,9 @@

      Extensibility and DSLs

      end {% endhighlight %} -

      The async: true option allows tests to run in parallel, using as many CPU cores as possible, while the assert functionality can introspect your code, providing great reports in case of failures. Those features are built using Elixir macros, making it possible to add new constructs as if they were part of the language itself.

      +

      The async: true option allows tests to run in parallel, using as many CPU cores as possible, while the assert functionality can introspect your code, providing great reports in case of failures.

      + +

      Other examples include using Elixir to write SQL queries, compiling a subset of Elixir to the GPU, and more.

      @@ -127,19 +151,18 @@

      Tooling features

      A growing ecosystem

      -

      Elixir ships with a great set of tools to ease development. Mix is a build tool that allows you to easily create projects, manage tasks, run tests and more:

      +

      Elixir ships with a great set of tools to ease development. Mix is a build tool that allows you to easily create projects, manage tasks, run tests and more:

      -{% highlight text %} -$ mix new my_app +
      $ mix new my_app
 $ cd my_app
 $ mix test
-.
+.
 
 Finished in 0.04 seconds (0.04s on load, 0.00s on tests)
-1 tests, 0 failures
-{% endhighlight %}
+1 test, 0 failures
+
      -

      Mix is also able to manage dependencies and integrates nicely with the Hex package manager, which provides dependency resolution and the ability to remotely fetch packages.

      +

      Mix also integrates with the Hex package manager for dependency management and hosting documentation for the whole ecosystem.

@@ -147,15 +170,18 @@

A growing ecosystem

Interactive development

-

Tools like IEx (Elixir's interactive shell) are able to leverage many aspects of the language and platform to provide auto-complete, debugging tools, code reloading, as well as nicely formatted documentation:

+

Tools like IEx (Elixir's interactive shell) leverage the language and platform to provide auto-complete, debugging tools, code reloading, as well as nicely formatted documentation:

{% highlight text %} $ iex Interactive Elixir - press Ctrl+C to exit (type h() ENTER for help) -iex> c "my_file.ex" # Compiles a file -iex> t Enum # Prints types defined in the module Enum -iex> h IEx.pry # Prints the documentation for IEx pry functionality +iex> h String.trim # Prints the documentation +iex> i "Hello, World" # Prints information about a data type +iex> break! String.trim/1 # Sets a breakpoint +iex> recompile # Recompiles the current project {% endhighlight %} + +

Code notebooks like Livebook allow you to interact with Elixir directly from your browser, including support for plotting, flowcharts, data tables, machine learning, and much more!

@@ -163,15 +189,14 @@

Interactive development

Erlang compatible

-

Elixir runs on the Erlang VM giving developers complete access to Erlang's ecosystem, used by companies like Heroku, Whatsapp, Klarna, Basho and many more to build distributed, fault-tolerant applications. An Elixir programmer can invoke any Erlang function with no runtime cost:

+

Elixir runs on the Erlang VM giving developers complete access to Erlang's ecosystem, used by companies like WhatsApp, Klarna, and many more to build distributed, fault-tolerant applications. An Elixir programmer can invoke any Erlang function with no runtime cost:

-{% highlight iex %} -iex> :crypto.md5("Using crypto from Erlang OTP") -<<192,223,75,115,...>> +{% highlight elixir %} +iex> :crypto.hash(:sha256, "Using crypto from Erlang OTP") +<<192, 223, 75, 115, ...>> {% endhighlight %} -

To learn more about Elixir, check our getting started guide. We also have online documentation available and a Crash Course for Erlang developers.

+

To learn more about Elixir, check our Getting Started guide.

- diff --git a/install.bat b/install.bat new file mode 100755 index 000000000..a6e88101c --- /dev/null +++ b/install.bat @@ -0,0 +1,207 @@ +@echo off +REM See latest version at: +REM https://github.com/elixir-lang/elixir-lang.github.com/blob/main/install.bat + +setlocal EnableDelayedExpansion + +set "otp_version=" +set "elixir_version=" +set "force=false" + +goto :main + +:usage +echo Usage: install.bat elixir@ELIXIR_VERSION otp@OTP_VERSION [options] +echo. +echo ELIXIR_VERSION can be X.Y.Z, latest, or main. +echo OTP_VERSION can be X.Y.Z or latest. +echo. +echo Options: +echo. +echo -f, --force Forces installation even if it was previously installed +echo -h, --help Prints this help +echo. +echo Examples: +echo. +echo install.bat elixir@1.16.3 otp@26.2.5.4 +echo install.bat elixir@latest otp@latest +echo install.bat elixir@main otp@latest +echo. +goto :eof + +:main +for %%i in (%*) do ( + set arg=%%i + + if "!arg:~0,7!" == "elixir@" ( + set "elixir_version=!arg:~7!" + ) else if "!arg:~0,4!" == "otp@" ( + set "otp_version=!arg:~4!" + ) else if "!arg!" == "-f" ( + set "force=true" + ) else if "!arg!" == "--force" ( + set "force=true" + ) else if "!arg!" == "-h" ( + call :usage + exit /b 0 + ) else if "!arg!" == "--help" ( + call :usage + exit /b 0 + ) else ( + echo error: unknown argument !arg! + exit /b 1 + ) +) + +if "%elixir_version%" == "" ( + call :usage + echo error: missing elixir@VERSION argument + exit /b 1 +) + +if "%otp_version%" == "" ( + call :usage + echo error: missing otp@VERSION argument + exit /b 1 +) + +if "!otp_version!" == "latest" ( + set "url=https://github.com/erlang/otp/releases/latest" + for /f "tokens=2 delims= " %%a in ('curl -fsS --head "!url!" ^| findstr /I "^location:"') do set url=%%a + set "otp_version=!url:*releases/tag/OTP-=!" +) + +if "!elixir_version!" == "latest" ( + set "url=https://github.com/elixir-lang/elixir/releases/latest" + for /f "tokens=2 delims= " %%a in ('curl -fsS --head "!url!" ^| findstr /I "^location:"') do set url=%%a + set "elixir_version=!url:*releases/tag/v=!" +) + +for /f "tokens=1 delims=." %%A in ("!otp_version!") do set "elixir_otp_release=%%A" +for /f "tokens=1,2 delims=." %%A in ("!elixir_version!") do set "elixir_major_minor=%%A.%%B" + +if "%elixir_major_minor%" == "1.14" ( + if %elixir_otp_release% GEQ 25 set "elixir_otp_release=25" +) else if "%elixir_major_minor%" == "1.15" ( + if %elixir_otp_release% GEQ 26 set "elixir_otp_release=26" +) else if "%elixir_major_minor%" == "1.16" ( + if %elixir_otp_release% GEQ 26 set "elixir_otp_release=26" +) else if "%elixir_major_minor%" == "1.17" ( + if %elixir_otp_release% GEQ 27 set "elixir_otp_release=27" +) else if "%elixir_major_minor%" == "1.18" ( + if %elixir_otp_release% GEQ 27 set "elixir_otp_release=27" +) else if "%elixir_major_minor%" == "1.19" ( + if %elixir_otp_release% GEQ 28 set "elixir_otp_release=28" +) else ( + if %elixir_otp_release% GEQ 28 set "elixir_otp_release=28" +) + +set "root_dir=%USERPROFILE%\.elixir-install" +set "tmp_dir=%root_dir%\tmp" +mkdir %tmp_dir% 2>nul +set "otp_dir=%root_dir%\installs\otp\%otp_version%" +set "elixir_dir=%root_dir%\installs\elixir\%elixir_version%-otp-%elixir_otp_release%" + +call :install_otp +if %errorlevel% neq 0 exit /b 1 + +set /p="checking OTP... "Previous Elixir versions are available in our [Releases](https://github.com/elixir-lang/elixir/releases) page. + + - Using [install scripts](#install-scripts) + + - Using [Scoop](https://scoop.sh/): + * Install Erlang: `scoop install erlang` + * Install Elixir: `scoop install elixir` + + - Using [Chocolatey](https://community.chocolatey.org/): + * Install Elixir (installs Erlang as a dependency): `choco install elixir` + +### Raspberry Pi and embedded devices + +To build and package an Elixir application, with the whole operating system, and burn that into a disk or deploy it overwhere, [check out the Nerves project](https://www.nerves-project.org). + +If you want to install Elixir as part of an existing Operating System, please follow the relevant steps above for your Operating System or install from precompiled/source. + +### Docker + +If you are familiar with Docker you can use the official Docker image to get started quickly with Elixir. + + * Enter interactive mode + * Run: `docker run -it --rm elixir` + * Enter bash within container with installed `elixir` + * Run: `docker run -it --rm elixir bash` + +The above will automatically point to the latest Erlang and Elixir available. For production usage, we recommend using [Hex.pm Docker images](https://hub.docker.com/r/hexpm/elixir), which are immutable and point to a specific Erlang and Elixir version. + +## Install scripts + +Elixir and Erlang/OTP can be quickly installed for macOS, Windows, or Ubuntu using an `install.sh`/`install.bat` script: + +If you are using bash (macOS/Ubuntu/Windows), run: + +```sh +curl -fsSO {{ site.url }}/install.sh +sh install.sh elixir@{{ stable.version }} otp@{{ stable.recommended_otp }} +installs_dir=$HOME/.elixir-install/installs +export PATH=$installs_dir/otp/{{ stable.recommended_otp }}/bin:$PATH +export PATH=$installs_dir/elixir/{{ stable.version }}-otp-{{ stable.otp_versions[0] }}/bin:$PATH +iex +``` + +If you are using PowerShell (Windows), run: + +```pwsh +curl.exe -fsSO {{ site.url }}/install.bat +.\install.bat elixir@{{ stable.version }} otp@{{ stable.recommended_otp }} +$installs_dir = "$env:USERPROFILE\.elixir-install\installs" +$env:PATH = "$installs_dir\otp\{{ stable.recommended_otp }}\bin;$env:PATH" +$env:PATH = "$installs_dir\elixir\{{ stable.version }}-otp-{{ stable.otp_versions[0] }}\bin;$env:PATH" +iex.bat +``` + +You may want to [set the $PATH environment variable](#setting-path-environment-variable) for your whole system. Use `install.sh --help` or `install.bat --help` to learn more about available arguments and options. + +## Precompiled package + +Elixir provides a precompiled package for every release. First [install Erlang](/install.html#installing-erlang) and then download the appropriate precompiled Elixir below. You can consult your Erlang/OTP version by running `erl -s halt`: + +{% for otp_version in stable.otp_versions %} + * [Elixir {{ stable.version }} on Erlang/OTP {{ otp_version }}](https://github.com/elixir-lang/elixir/releases/download/v{{ stable.version }}/elixir-otp-{{ otp_version }}.zip){% endfor %} + +Once you download the release, unpack it, and you are ready to run the `elixir` and `iex` commands from the `bin` directory. However, we recommend you to [add Elixir's bin path to your PATH environment variable](#setting-path-environment-variable) to ease development. + +### Mirrors and nightly builds + +The links above point directly to the GitHub release. We also host and mirror precompiled packages and nightly builds globally via `builds.hex.pm` using the following URL scheme: + + https://builds.hex.pm/builds/elixir/${ELIXIR_VERSION}-otp-${OTP_VERSION}.zip + +For example, to use Elixir v1.13.3 with Erlang/OTP 24.x, use: + + https://builds.hex.pm/builds/elixir/v1.13.3-otp-24.zip + +To use nightly for a given Erlang/OTP version (such as 25), use: + + https://builds.hex.pm/builds/elixir/main-otp-25.zip + +For a list of all builds, use: + + https://builds.hex.pm/builds/elixir/builds.txt + +## Compiling from source + +You can download and compile Elixir in few steps. The first one is to [install Erlang](/install.html#installing-erlang). You will also need [make](https://www.gnu.org/software/make/) available. + +Next you should download source code ([.zip](https://github.com/elixir-lang/elixir/archive/v{{ stable.version }}.zip), [.tar.gz](https://github.com/elixir-lang/elixir/archive/v{{ stable.version }}.tar.gz)) of the [latest release](https://github.com/elixir-lang/elixir/releases/tag/v{{ stable.version }}), unpack it and then run `make` inside the unpacked directory (note: if you are running on Windows, [read this page on setting up your environment for compiling Elixir](https://github.com/elixir-lang/elixir/wiki/Windows)). + +After compiling, you are ready to run the elixir and `iex` commands from the bin directory. It is recommended that you [add Elixir's bin path to your PATH environment variable](#setting-path-environment-variable) to ease development. - * Web installer - * [Download the installer](http://s3.hex.pm/elixir-websetup.exe) - * Click next, next, ..., finish - * Chocolatey - * `cinst elixir` +In case you are feeling a bit more adventurous, you can also compile from main: -Those distributions will likely install Erlang automatically for you too. In case they don't, check the [Installing Erlang](/install.html#4-installing-erlang) section below. +```bash +$ git clone https://github.com/elixir-lang/elixir.git +$ cd elixir +$ make clean compile +``` -### 2 Precompiled package +## Installing Erlang -Elixir provides a precompiled package for every release. First [install Erlang](/install.html#4-installing-erlang) and then download and unzip the [Precompiled.zip file for the latest release](https://github.com/elixir-lang/elixir/releases/). +The only prerequisite for Elixir is Erlang, version {{ stable.minimum_otp }} or later. When installing Elixir, Erlang is generally installed automatically for you. However, if you want to install Erlang manually, you might check: -Once the release is unpacked, you are ready to run the `elixir` and `iex` commands from the `bin` directory. It is recommended that you add Elixir's `bin` path to your PATH environment variable to ease development. + * [Source code distribution and Windows installers from Erlang's official website](http://www.erlang.org/downloads.html) + * [Precompiled packages for some Unix-like installations](https://www.erlang-solutions.com/resources/download.html) + * [A general list of installation methods from the Riak documentation](https://docs.riak.com/riak/kv/latest/setup/installing/source/erlang/) - $ export PATH="$PATH:/path/to/elixir/bin" +After Erlang is installed, you should be able to open up the command line (or command prompt) and check the Erlang version by typing `erl -s erlang halt`. You will see some information similar to: -### 3 Compiling from source (Unix and MinGW) + Erlang/OTP {{ stable.minimum_otp }} [64-bit] [smp:2:2] [...] -You can download and compile Elixir in few steps. The first one is to [install Erlang](/install.html#4-installing-erlang). +Notice that depending on how you installed Erlang, Erlang binaries might not be available in your PATH. Be sure to have Erlang binaries in your [PATH](https://en.wikipedia.org/wiki/Environment_variable), otherwise Elixir won't work! -Next you should download the [latest release](https://github.com/elixir-lang/elixir/releases/), unpack it and then run `make` inside the unpacked directory (note: if you are running on Windows, [read this page on setting up your environment for compiling Elixir](https://github.com/elixir-lang/elixir/wiki/Windows)). +## Setting PATH environment variable -After compiling, you are ready to run the `elixir` and `iex` commands from the `bin` directory. It is recommended that you add Elixir's `bin` path to your [PATH](http://en.wikipedia.org/wiki/Environment_variable) environment variable to ease development: +It is highly recommended to add Elixir's bin path to your PATH environment variable to ease development. - $ export PATH="$PATH:/path/to/elixir/bin" +On Windows, there are [instructions for different versions](http://www.computerhope.com/issues/ch000549.htm) explaining the process. -In case you are feeling a bit more adventurous, you can also compile from master: +On Unix systems, you need to [find your shell profile file](https://unix.stackexchange.com/a/117470/101951), and then add to the end of this file the following line reflecting the path to your Elixir installation: - $ git clone https://github.com/elixir-lang/elixir.git - $ cd elixir - $ make clean test +```bash +export PATH="$PATH:/path/to/elixir/bin" +``` -If the tests pass, you are ready to go. Otherwise, feel free to open an issue [in the issues tracker on Github](https://github.com/elixir-lang/elixir). +## Asking questions -### 4 Installing Erlang +After Elixir is up and running, it is common to have questions as you learn and use the language. There are many places where you can ask questions, here are some of them: -The only prerequisite for Elixir is Erlang, version 17.0 or later, which can be easily installed with [Precompiled packages](https://www.erlang-solutions.com/downloads/download-erlang-otp). In case you want to install it directly from source, it can be found on [the Erlang website](http://www.erlang.org/download.html) or by following the excellent tutorial available in the [Riak documentation](http://docs.basho.com/riak/1.3.0/tutorials/installation/Installing-Erlang/). + * [#elixir on irc.libera.chat](irc://irc.libera.chat/elixir) + * [Elixir Forum](http://elixirforum.com) + * [Elixir on Slack](https://elixir-slack.community) + * [Elixir on Discord](https://discord.gg/elixir) + * [elixir tag on StackOverflow](https://stackoverflow.com/questions/tagged/elixir) -For Windows developers, we recommend the precompiled packages. Those on a Unix platform can probably get Erlang installed via one of the many package distribution tools. +When asking questions, remember these two tips: -After Erlang is installed, you should be able to open up the command line (or command prompt) and check the Erlang version by typing `erl`. You will see some information as follows: + * Instead of asking "how to do X in Elixir", ask "how to solve Y in Elixir". In other words, don't ask how to implement a particular solution, instead describe the problem at hand. Stating the problem gives more context and less bias for a correct answer. - Erlang/OTP 17 (erts-6) [64-bit] [smp:2:2] [async-threads:0] [hipe] [kernel-poll:false] + * In case things are not working as expected, please include as much information as you can in your report, for example: your Elixir version, the code snippet and the error message alongside the error stacktrace. -Notice that depending on how you installed Erlang, Erlang binaries won't be available in your PATH. Be sure to have Erlang binaries in your [PATH](http://en.wikipedia.org/wiki/Environment_variable), otherwise Elixir won't work! +Enjoy! diff --git a/install.sh b/install.sh new file mode 100755 index 000000000..f1647a15f --- /dev/null +++ b/install.sh @@ -0,0 +1,298 @@ +#!/bin/sh +# See latest version at: +# https://github.com/elixir-lang/elixir-lang.github.com/blob/main/install.sh + +set -eu + +otp_version= +elixir_version= +force=false + +usage() { + cat<&2 + exit 1 + ;; + esac + done + + if [ -z "${elixir_version}" ]; then + usage + echo "error: missing elixir@VERSION argument" + exit 1 + fi + + if [ -z "${otp_version}" ]; then + usage + echo "error: missing otp@VERSION argument" + exit 1 + fi + + root_dir="$HOME/.elixir-install" + tmp_dir="$root_dir/tmp" + mkdir -p "$tmp_dir" + + if [ "${otp_version}" = latest ]; then + url=$(curl -fsS --head https://github.com/erlef/otp_builds/releases/latest | grep -i '^location:' | awk '{print $2}' | tr -d '\r\n') + tag=$(basename "$url") + otp_version="${tag#OTP-}" + fi + + if [ "${elixir_version}" = latest ]; then + url=$(curl -fsS --head https://github.com/elixir-lang/elixir/releases/latest | grep -i '^location:' | awk '{print $2}' | tr -d '\r\n') + tag=$(basename "$url") + elixir_version="${tag#v}" + fi + + case "${otp_version}" in + master|maint*) + branch_version=$(curl -fsS https://raw.githubusercontent.com/erlang/otp/refs/heads/${otp_version}/OTP_VERSION | tr -d '\n') + elixir_otp_release="${branch_version%%.*}" + ;; + *) + elixir_otp_release="${otp_version%%.*}" + ;; + esac + + case "$elixir_version" in + 1.14.*) + [ "${elixir_otp_release}" -ge 25 ] && elixir_otp_release=25 + ;; + 1.15.*|1.16.*) + [ "${elixir_otp_release}" -ge 26 ] && elixir_otp_release=26 + ;; + 1.17.*|1.18.*) + [ "${elixir_otp_release}" -ge 27 ] && elixir_otp_release=27 + ;; + 1.19.*) + [ "${elixir_otp_release}" -ge 28 ] && elixir_otp_release=28 + ;; + *) + [ "${elixir_otp_release}" -ge 28 ] && elixir_otp_release=28 + ;; + esac + + otp_dir="$root_dir/installs/otp/$otp_version" + elixir_dir="${root_dir}/installs/elixir/${elixir_version}-otp-${elixir_otp_release}" + + if unzip_available; then + install_otp & + pid_otp=$! + + install_elixir & + pid_elixir=$! + + wait $pid_otp + wait $pid_elixir + else + # if unzip is missing (e.g. official docker ubuntu image), install otp and elixir + # serially because we unzip elixir using OTP zip:extract/2. + install_otp + install_elixir + fi + + printf "checking OTP... " + export PATH="$otp_dir/bin:$PATH" + erl -noshell -eval 'io:put_chars(erlang:system_info(otp_release) ++ " ok\n"), halt().' + + printf "checking Elixir... " + "$elixir_dir/bin/elixir" -e 'IO.puts(System.version() <> " ok")' + + export PATH="$elixir_dir/bin:$PATH" +cat</dev/null 2>&1 +} + +main "$@" diff --git a/js/icons/fonts/LICENSE.txt b/js/icons/fonts/LICENSE.txt new file mode 100644 index 000000000..259b43d14 --- /dev/null +++ b/js/icons/fonts/LICENSE.txt @@ -0,0 +1,9 @@ +(c) 2012-2014 GitHub + +When using the GitHub logos, be sure to follow the GitHub logo guidelines (https://github.com/logos) + +Font License: SIL OFL 1.1 (http://scripts.sil.org/OFL) +Applies to all font files + +Code License: MIT (http://choosealicense.com/licenses/mit/) +Applies to all other files diff --git a/js/icons/fonts/icons.eot b/js/icons/fonts/icons.eot new file mode 100644 index 000000000..1d30e66c2 Binary files /dev/null and b/js/icons/fonts/icons.eot differ diff --git a/js/icons/fonts/icons.svg b/js/icons/fonts/icons.svg new file mode 100644 index 000000000..4b4d09d85 --- /dev/null +++ b/js/icons/fonts/icons.svg @@ -0,0 +1,30 @@ + + + + + +{ + "fontFamily": "icons", + "majorVersion": 1, + "minorVersion": 0, + "version": "Version 1.0", + "fontId": "icons", + "psName": "icons", + "subFamily": "Regular", + "fullName": "icons", + "description": "Font generated by IcoMoon." +} + + + + + + + + + + + + + + \ No newline at end of file diff --git a/js/icons/fonts/icons.ttf b/js/icons/fonts/icons.ttf new file mode 100644 index 000000000..2b8649e44 Binary files /dev/null and b/js/icons/fonts/icons.ttf differ diff --git a/js/icons/fonts/icons.woff b/js/icons/fonts/icons.woff new file mode 100644 index 000000000..383494c22 Binary files /dev/null and b/js/icons/fonts/icons.woff differ diff --git a/js/icons/ie7/ie7.css b/js/icons/ie7/ie7.css new file mode 100644 index 000000000..ce158f6f9 --- /dev/null +++ b/js/icons/ie7/ie7.css @@ -0,0 +1,18 @@ +.icon-chevron-up { + *zoom: expression(this.runtimeStyle['zoom'] = '1', this.innerHTML = ''); +} +.icon-code { + *zoom: expression(this.runtimeStyle['zoom'] = '1', this.innerHTML = ''); +} +.icon-file-code { + *zoom: expression(this.runtimeStyle['zoom'] = '1', this.innerHTML = ''); +} +.icon-file-text { + *zoom: expression(this.runtimeStyle['zoom'] = '1', this.innerHTML = ''); +} +.icon-link { + *zoom: expression(this.runtimeStyle['zoom'] = '1', this.innerHTML = ''); +} +.icon-link-external { + *zoom: expression(this.runtimeStyle['zoom'] = '1', this.innerHTML = ''); +} diff --git a/js/icons/ie7/ie7.js b/js/icons/ie7/ie7.js new file mode 100644 index 000000000..d3b490ab7 --- /dev/null +++ b/js/icons/ie7/ie7.js @@ -0,0 +1,37 @@ +/* To avoid CSS expressions while still supporting IE 7 and IE 6, use this script */ +/* The script tag referencing this file must be placed before the ending body tag. */ + +/* Use conditional comments in order to target IE 7 and older: + + + +*/ + +(function() { + function addIcon(el, entity) { + var html = el.innerHTML; + el.innerHTML = '' + entity + '' + html; + } + var icons = { + 'icon-chevron-up': '', + 'icon-code': '', + 'icon-file-code': '', + 'icon-file-text': '', + 'icon-link': '', + 'icon-link-external': '', + '0': 0 + }, + els = document.getElementsByTagName('*'), + i, c, el; + for (i = 0; ; i += 1) { + el = els[i]; + if(!el) { + break; + } + c = el.className; + c = c.match(/icon-[^\s'"]+/); + if (c && icons[c[0]]) { + addIcon(el, icons[c[0]]); + } + } +}()); diff --git a/js/icons/selection.json b/js/icons/selection.json new file mode 100644 index 000000000..6126d2489 --- /dev/null +++ b/js/icons/selection.json @@ -0,0 +1,203 @@ +{ + "IcoMoonType": "selection", + "icons": [ + { + "icon": { + "paths": [ + "M320 256l-320 320 128 128 192-192 192 192 128-128-320-320z" + ], + "width": 640, + "attrs": [], + "isMulticolor": false, + "tags": [ + "chevron-up" + ], + "defaultCode": 61602, + "grid": 16 + }, + "attrs": [], + "properties": { + "id": 25, + "order": 1, + "prevSize": 32, + "name": "chevron-up", + "code": 61602 + }, + "setIdx": 0, + "iconIdx": 25 + }, + { + "icon": { + "paths": [ + "M608 192l-96 96 224 224-224 224 96 96 288-320-288-320zM288 192l-288 320 288 320 96-96-224-224 224-224-96-96z" + ], + "width": 896, + "attrs": [], + "isMulticolor": false, + "tags": [ + "code" + ], + "defaultCode": 61535, + "grid": 16 + }, + "attrs": [], + "properties": { + "id": 32, + "order": 6, + "prevSize": 32, + "name": "code", + "code": 61535 + }, + "setIdx": 0, + "iconIdx": 32 + }, + { + "icon": { + "paths": [ + "M288 384l-160 160 160 160 64-64-96-96 96-96-64-64zM416 448l96 96-96 96 64 64 160-160-160-160-64 64zM576 64h-576v896h768v-704l-192-192zM704 896h-640v-768h448l192 192v576z" + ], + "width": 768, + "attrs": [], + "isMulticolor": false, + "tags": [ + "file-code" + ], + "defaultCode": 61456, + "grid": 16 + }, + "attrs": [], + "properties": { + "id": 53, + "order": 7, + "prevSize": 32, + "name": "file-code", + "code": 61456 + }, + "setIdx": 0, + "iconIdx": 53 + }, + { + "icon": { + "paths": [ + "M448 256h-320v64h320v-64zM576 64h-576v896h768v-704l-192-192zM704 896h-640v-768h448l192 192v576zM128 768h512v-64h-512v64zM128 640h512v-64h-512v64zM128 512h512v-64h-512v64z" + ], + "width": 768, + "attrs": [], + "isMulticolor": false, + "tags": [ + "file-text" + ], + "defaultCode": 61457, + "grid": 16 + }, + "attrs": [], + "properties": { + "id": 60, + "order": 9, + "prevSize": 32, + "name": "file-text", + "code": 61457 + }, + "setIdx": 0, + "iconIdx": 60 + }, + { + "icon": { + "paths": [ + "M768 256h-138c48 32 93 89 107 128h30c65 0 128 64 128 128s-65 128-128 128h-192c-63 0-128-64-128-128 0-23 7-45 18-64h-137c-5 21-8 42-8 64 0 128 127 256 255 256s65 0 193 0 256-128 256-256-128-256-256-256zM287 640h-30c-65 0-128-64-128-128s65-128 128-128h192c63 0 128 64 128 128 0 23-7 45-18 64h137c5-21 8-42 8-64 0-128-127-256-255-256s-65 0-193 0-256 128-256 256 128 256 256 256h138c-48-32-93-89-107-128z" + ], + "attrs": [], + "isMulticolor": false, + "tags": [ + "link" + ], + "defaultCode": 61532, + "grid": 16 + }, + "attrs": [], + "properties": { + "id": 95, + "order": 2, + "prevSize": 32, + "name": "link", + "code": 61532 + }, + "setIdx": 0, + "iconIdx": 95 + }, + { + "icon": { + "paths": [ + "M640 768h-512v-510.094l128-1.906v-128h-256v768h768v-320h-128v192zM384 128l128 128-192 192 128 128 192-192 128 128v-384h-384z" + ], + "width": 768, + "attrs": [], + "isMulticolor": false, + "tags": [ + "link-external" + ], + "defaultCode": 61567, + "grid": 16 + }, + "attrs": [], + "properties": { + "id": 96, + "order": 3, + "prevSize": 32, + "name": "link-external", + "code": 61567 + }, + "setIdx": 0, + "iconIdx": 96 + } + ], + "height": 1024, + "metadata": { + "name": "icons" + }, + "preferences": { + "showGlyphs": true, + "showQuickUse": true, + "showQuickUse2": true, + "showSVGs": true, + "fontPref": { + "prefix": "icon-", + "metadata": { + "fontFamily": "icons", + "majorVersion": 1, + "minorVersion": 0 + }, + "metrics": { + "emSize": 1024, + "baseline": 6.25, + "whitespace": 50 + }, + "autoHost": true, + "ie7": true, + "includeMetadata": true, + "embed": true, + "showSelector": true, + "showMetrics": true, + "showMetadata": true, + "showVersion": true, + "resetPoint": 58880, + "selector": "", + "classSelector": ".iconx" + }, + "imagePref": { + "prefix": "icon-", + "png": true, + "useClassSelector": true, + "color": 4473924, + "bgColor": 16777215, + "classSelector": ".icon", + "height": 32 + }, + "historySize": 100, + "showCodes": true, + "quickUsageToken": { + "elixir-langorg": "NTcyMGFiMjE2N2UwMWNiZTQ2YzU3YTM2OWEzMGE5YTcjMSMxNDIzNjUwNjQxIyMj" + }, + "showLiga": false + } +} \ No newline at end of file diff --git a/js/icons/style.css b/js/icons/style.css new file mode 100644 index 000000000..59992103b --- /dev/null +++ b/js/icons/style.css @@ -0,0 +1,50 @@ +/* Elixir-lang.org NOTE: I have updated the @font-face declaration, so if you update the fonts, please update this. */ +@font-face { + font-family: 'icons'; + src: url('fonts/icons.eot'); /* IE9 Compat Modes */ + src: url('fonts/icons.eot?#iefix') format('embedded-opentype'), /* IE6-IE8 */ + url('fonts/icons.woff') format('woff'), /* Pretty Modern Browsers */ + url('fonts/icons.ttf') format('truetype'), /* Safari, Android, iOS */ + url('fonts/icons.svg#icons') format('svg'); /* Legacy iOS */ + font-weight: normal; + font-style: normal; +} + +[class^="icon-"], [class*=" icon-"] { + font-family: 'icons'; + speak: none; + font-style: normal; + font-weight: normal; + font-variant: normal; + text-transform: none; + line-height: 1; + + /* Better Font Rendering =========== */ + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.icon-chevron-up:before { + content: "\f0a2"; +} + +.icon-code:before { + content: "\f05f"; +} + +.icon-file-code:before { + content: "\f010"; +} + +.icon-file-text:before { + content: "\f011"; +} + +.icon-link:before { + content: "\f05c"; +} + +.icon-link-external:before { + content: "\f07f"; +} + diff --git a/js/index.js b/js/index.js new file mode 100644 index 000000000..cb605e3a8 --- /dev/null +++ b/js/index.js @@ -0,0 +1,163 @@ +// https://github.com/ghiculescu/jekyll-table-of-contents +$.fn.toc = function(options) { + var defaults = { + title: '', + minimumHeaders: 3, + headers: 'h1, h2, h3, h4, h5, h6', + listType: 'ol', // values: [ol|ul] + + linkHeader: true, + linkHere: false, + linkHereText: '', + linkHereTitle: 'Link here', + backToTop: false, + backToTopId: '', + backToTopText: '', + backToTopTitle: 'Back to top', + backToTopDisplay: 'always', // values: [always|highest] + }, + settings = $.extend(defaults, options); + + var headers = $(settings.headers).filter(function() { + // get all headers with an ID + var previousSiblingName = $(this).prev().attr( "name" ); + if (!this.id && previousSiblingName) { + this.id = $(this).attr( "id", previousSiblingName.replace(/\./g, "-") ); + } + return this.id; + }), output = $(this); + + if (!headers.length || headers.length < settings.minimumHeaders || !output.length) { + return; + } + + var get_level = function(ele) { return parseInt(ele.nodeName.replace("H", ""), 10); } + var highest_level = headers.map(function(_, ele) { return get_level(ele); }).get().sort()[0]; + + var level = get_level(headers[0]), + this_level, + html = settings.title + ' <'+settings.listType+' class="jekyll-toc">'; + + var back_to_top = function(id) { + return ''+settings.backToTopText+''; + } + + var link_here = function(id) { + return ''+settings.linkHereText+''; + } + + function fixedEncodeURIComponent (str) { + return encodeURIComponent(str).replace(/[!'()*]/g, function(c) { + return '%' + c.charCodeAt(0).toString(16); + }); + } + + function force_update_hash(hash) { + if ( window.location.hash == hash ) { + window.location.hash = ''; + } + window.location.hash = hash; + + } + + $(headers).each(function(_, header) { + this_level = get_level(header); + var header_id = $(header).attr('id'); + if (this_level === level) // same level as before; same indenting + html += "
  • " + header.innerHTML + ""; + else if (this_level <= level){ // higher level than before; end parent ol + for(i = this_level; i < level; i++) { + html += "
    • " + } + html += "
  • " + header.innerHTML + ""; + } + else if (this_level > level) { // lower level than before; expand the previous to contain a ol + for(i = this_level; i > level; i--) { + html += "<"+settings.listType+">
  • " + } + html += "" + header.innerHTML + ""; + } + level = this_level; // update for the next one + + // add links at the end (so we don't pulute header.innerHTML) + $(header).addClass('jekyll-toc-top-level-header').wrapInner('').append( link_here(header_id) ); + if (settings.backToTop) { + switch(settings.backToTopDisplay){ + case 'highest': + if ( this_level === highest_level ) { + $(header).append( back_to_top(settings.backToTopId) ); + } + break; + case 'always': + default: + $(header).append( back_to_top(settings.backToTopId) ); + } + } + + if (settings.linkHeader) { + $(header).addClass('jekyll-toc-header'); + $(header).children('span.jekyll-toc-wrapper').on( 'click', function( ) { + force_update_hash(fixedEncodeURIComponent(header_id)); + }); + } + }); + + html += ""; + + output.html(html) +}; + +// https://css-tricks.com/snippets/jquery/shuffle-dom-elements/ +$.fn.shuffle = function() { + var allElems = this.get(), + getRandom = function(max) { + return Math.floor(Math.random() * max); + }, + shuffled = $.map(allElems, function(){ + var random = getRandom(allElems.length), + randEl = $(allElems[random]).clone(true)[0]; + allElems.splice(random, 1); + return randEl; + }); + + this.each(function(i){ + $(this).replaceWith($(shuffled[i])); + }); + + return $(shuffled); +}; + +// use plugins +$(document).ready(function() { + $('.toc').toc({ + title: '', + listType: 'ol', + minimumHeaders: 2, + headers: 'h2, h3, h4, h5, h6', + linkHere: true, + linkHereTitle: 'Link here', + backToTop: true, + backToTopId: 'toc', + backToTopTitle: 'Back to Table of Contents', + }); + + $('.jekyll-toc-header a.jekyll-toc-link-here span.jekyll-toc-icon').addClass('icon icon-link'); + $('.jekyll-toc-header a.jekyll-toc-back-to-top span.jekyll-toc-icon').addClass('icon icon-chevron-up'); + + $('#top-banner .close').click(function() { + $(this).parent().slideUp(200); + document.cookie = 'topBannerDisabled=true'; + }) + + $("#shuffled-cases").children().shuffle() +}); + +// Automatically redirect to https +(function (){ + const location = window.location.href; + const localhost = /:\/\/(?:localhost|127.0.0.1|::1)/ + + if (location.startsWith("http://") && !localhost.test(location)) { + window.location.href = location.replace("http://", "https://"); + } +})(); \ No newline at end of file diff --git a/js/toc/README.md b/js/toc/README.md new file mode 100644 index 000000000..2d9412d0f --- /dev/null +++ b/js/toc/README.md @@ -0,0 +1,132 @@ +# jekyll-table-of-contents + +A simple JavaScript table of contents generator. Works well with [jekyll](https://github.com/mojombo/jekyll) static sites. + +## Usage + +### Basic Usage + +The script requires jQuery. First, reference `toc.js` in templates where you would like to add the table of content. +Then, create an HTML element wherever you want your table of contents to appear: + +```html +
    +``` + +Finally, call the `.toc()` function when the DOM is ready: + +```html + +``` + +If you use redcarpet, you need to have the option `with_toc_data` in order to add HTML anchors to each header: +```yaml +markdown: redcarpet +redcarpet: + extensions: [with_toc_data] +``` + +If you use rdiscount, enable the following option in order to generate the TOC: +```yaml +markdown: rdiscount +rdiscount: + extensions: + - generate_toc +``` + +### How It Works + +The script works by looking for headers (h1, h2, h3, h4, h5, h6) which have an `id`. +An id is added automatically if you're using Jekyll and [Markdown](http://daringfireball.net/projects/markdown/syntax#header). + +The table of contents automatically handles nesting of headers. For example, this Markdown post: + + ## Title + ## Page 1 + ### Note on Paragraph 3 + ## Page 2 + ### Note on Paragraph 2 + ### Note on Paragraph 4 + +Will render this table of contents: + + 1. Title + 2. Page 1 + a. Note on Paragraph 3 + 3. Page 2 + a. Note on Paragraph 2 + b. Note on Paragraph 4 + +### Configuration + +#### List Type +By default the table of contents is rendered as an `
      `, so you can change the number formatting using CSS. +However, you can use the `
        ` tag, using the `listType` option: + +```javascript +$('#toc').toc({ listType: 'ul' }); +``` + +#### Header Styling +The script also adds an `` tag next to each header. This uses the class `icon-arrow-up`, which if you're using [Bootstrap](http://twitter.github.io/bootstrap/), will be an arrow pointing to the top of the page. +Clicking that arrow will scroll you to the top, while clicking on a header will get a permanent link to that particular header (using `window.location.hash`). + +If you don't want this feature, add this setting: + +```javascript +$('#toc').toc({ noBackToTopLinks: true }); +``` + +Otherwise, you can use the stylesheet below to have the icon and the header aligned nicely: + +```css +.clickable-header { + cursor:pointer; +} +.clickable-header:hover { + text-decoration:underline; +} +.top-level-header { + display:inline; +} +.back-to-top { + margin-left:5px; + cursor:pointer; +} +``` + +#### Headers Used +By default the table of content is displayed when at least 3 headers are found. +You can customize the minimum number of headers required with this setting: + +```javascript +$('#toc').toc({ minimumHeaders: 2 }); +``` + +And you can also select which headers you want to link to. By default `h1, h2, h3, h4, h5, h6` are displayed, but changing the `headers` setting lets you tweak it: + +```javascript +$('#toc').toc({ headers: 'h3, h4, h5, h6' }); +$('#toc').toc({ headers: '.content h1, .content h2, .content h3, .content h4, .content h5, .content h6' }); +``` + +#### Effects +Finally, you can also change the way the toc is displayed, choosing a `slideShow` or a `fadeIn` effect instead of `show`: + +```javascript +$('#toc').toc({ showEffect: 'slideDown' }); +``` + +Otherwise, to deactivate the effect, set it up like this: + +```javascript +$('#toc').toc({ showSpeed: 0 }); +``` + +## Copyright + +See LICENSE.txt for further details. But basically, do what you like with this. diff --git a/js/toc/toc.js b/js/toc/toc.js deleted file mode 100644 index 7b7e3e9e8..000000000 --- a/js/toc/toc.js +++ /dev/null @@ -1,63 +0,0 @@ -// https://github.com/ghiculescu/jekyll-table-of-contents -(function($){ - $.fn.toc = function(options) { - var defaults = { - noBackToTopLinks: false, - title: 'Jump to...', - listType: 'ol', // values: [ol|ul] - showSpeed: 'slow' - }, - settings = $.extend(defaults, options); - - // I modified the single line below to exlude h1 header - // --alco - var headers = $('h2, h3, h4, h5, h6').filter(function() { - // get all headers with an ID - return this.id; - }), output = $(this); - // Show the toc when there are at least 2 headers - // --alco - if (!headers.length || headers.length < 2 || !output.length) { - return; - } - - var get_level = function(ele) { return parseInt(ele.nodeName.replace("H", ""), 10); } - var highest_level = headers.map(function(_, ele) { return get_level(ele); }).get().sort()[0]; - var return_to_top = ' '; - - var level = get_level(headers[0]), - this_level, - html = settings.title + " <"+settings.listType+">"; - headers.on('click', function() { - if (!settings.noBackToTopLinks) { - window.location.hash = this.id; - } - }) - .addClass('clickable-header') - .each(function(_, header) { - this_level = get_level(header); - if (!settings.noBackToTopLinks && this_level === highest_level) { - $(header).addClass('top-level-header').after(return_to_top); - } - if (this_level === level) // same level as before; same indenting - html += "
      • " + header.innerHTML + ""; - else if (this_level < level) // higher level than before; end parent ol - html += "
      • " + header.innerHTML + ""; - else if (this_level > level) // lower level than before; expand the previous to contain a ol - html += "<"+settings.listType+">
      • " + header.innerHTML + ""; - level = this_level; // update for the next one - }); - html += ""; - if (!settings.noBackToTopLinks) { - $(document).on('click', '.back-to-top', function() { - $(window).scrollTop(0); - window.location.hash = ''; - }); - } - if (0 !== settings.showSpeed) { - output.hide().html(html).show(settings.showSpeed); - } else { - output.html(html); - } - }; -})(jQuery); diff --git a/learning.markdown b/learning.markdown new file mode 100644 index 000000000..c78d1ba5a --- /dev/null +++ b/learning.markdown @@ -0,0 +1,303 @@ +--- +title: "Learning resources" +section: learning +layout: default +image: /images/social/elixir-og-card.jpg +--- + +# Learning + +{% include toc.html %} + +Elixir's official documentation includes a [Getting Started guide](https://hexdocs.pm/elixir/introduction.html) to learn more about Elixir's foundations. Later on, it explores how to build projects with [Mix and OTP](https://hexdocs.pm/elixir/introduction-to-mix.html). Elixir also includes [extensive API documentation](/docs.html). + +The Elixir Community has also produced plenty of resources to learn the language from different backgrounds and other perspectives. We list some of them below. We are sure you will find a resource that suits your pace and goals. + +## Books + +

        Elixir in Action

        + +Elixir in Action cover + +Elixir in Action is a tutorial book that aims to bring developers new to Elixir and Erlang to the point where they can develop complex systems on their own. No knowledge about Elixir, Erlang, or functional programming is required, but it is assumed that a reader has a few years of production experience using mainstream OO languages, for example C#, Java, Python, or Ruby. + +The book starts with a basic introduction to the Elixir language and functional programming idioms. The central part of the book deals with Erlang VM and OTP, discussing topics such as concurrent programming, fault-tolerance, and distributed systems. Finally, you'll learn how to package your code into components, create a standalone deployable release, and troubleshoot the running system. The theory is demonstrated through a simplistic example that is gradually expanded throughout the book into a fully standalone releasable system. + +
        + +

        Programming Elixir 1.6

        + +Programming Elixir cover + +This book is the introduction to Elixir for experienced programmers, completely updated for Elixir 1.6 and beyond. Explore functional programming without the academic overtones (tell me about monads just one more time). Create concurrent applications, but get them right without all the locking and consistency headaches. + +Meet Elixir, a modern, functional, concurrent language built on the rock-solid Erlang VM. Elixir's pragmatic syntax and built-in support for metaprogramming will make you productive and keep you interested for the long haul. Maybe the time is right for the Next Big Thing. Maybe it's Elixir. + +
        + +

        Adopting Elixir

        + +Programming Elixir cover + +Adoption is more than programming. Elixir is an exciting new language, but to successfully get your application from start to finish, you're going to need to know more than just the language. You need the case studies and strategies in this book. + +Learn the best practices for the whole life of your application, from design and team-building, to managing stakeholders, to deployment and monitoring. Go beyond the syntax and the tools to learn the techniques you need to develop your Elixir application from concept to production. + +
        + +

        Joy of Elixir

        + +Joy of Elixir + +Joy of Elixir is a gentle introduction to programming, aimed at people who already know some things about computers, but who have little-to-no programming experience. + +This book will teach you the core concepts of the Elixir programming language in a fun and enjoyable way. If you're completely new to programming and you want to learn how to make a computer do things using the power of programming and you want to experience some joy while doing it, then read this book! + +
        + +

        Learn Functional Programming With Elixir

        + +Learn Functional Programming with Elixir cover + +Elixir's straightforward syntax and this guided tour give you a clean, simple path to learn modern functional programming techniques. No previous functional programming experience required! This book walks you through the right concepts at the right pace, as you explore immutable values and explicit data transformation, functions, modules, recursive functions, pattern matching, high-order functions, polymorphism, and failure handling, all while avoiding side effects. Don't board the Elixir train with an imperative mindset! To get the most out of functional languages, you need to think functionally. This book will get you there. + +
        + +

        The Toy Robot Walkthrough

        + +Toy Robot + +The Toy Robot is a common interview exercise for new programmers. This short book will take you through how to implement it in Elixir in a BDD-style, with some great explanations and imagery along the way. + +If you're a new Elixir developer who's gone through some basic Elixir tutorials and you're looking for the next thing to build your skills, this book is a great start. It covers the Toy Robot exercise from start to finish, testing with Elixir features such as ExUnit and Doctests along the way. + +
        + +

        Elixir Succinctlyfree

        + +Elixir Succinctly + +Elixir Succinctly is a free ebook to start learning Elixir. It covers the installation and the first steps with the language and the syntax. It then describes the Erlang/OTP platform, describing messages, processes, and GenServer. The final part covers the building of a sample Elixir application. + +
        + +## In-depth books + +

        Metaprogramming Elixir

        + +Metaprogramming Elixir cover + +Write code that writes code with Elixir macros. Macros make metaprogramming possible and define the language itself. In this book, you'll learn how to use macros to extend the language with fast, maintainable code and share functionality in ways you never thought possible. You'll discover how to extend Elixir with your own first-class features, optimize performance, and create domain-specific languages. + +
        + +

        Designing Elixir Systems with OTP

        + +Designing Elixir Systems with OTP cover + +You know how to code in Elixir; now learn to think in it. Learn to design libraries with intelligent layers that shape the right data structures, flow from one function into the next, and present the right APIs. Embrace the same OTP that's kept our telephone systems reliable and fast for over 30 years. Move beyond understanding the OTP functions to knowing what's happening under the hood, and why that matters. Using that knowledge, instinctively know how to design systems that deliver fast and resilient services to your users, all with an Elixir focus. + +
        + +

        Concurrent Data Processing in Elixir

        + +Concurrent Data Processing cover + +Learn different ways of writing concurrent code in Elixir and increase your application’s performance, without sacrificing scalability or fault-tolerance. Most projects benefit from running background tasks and processing data concurrently, but the world of OTP and various libraries can be challenging. Which Supervisor and what strategy to use? What about GenServer? Maybe you need back-pressure, but is GenStage, Flow, or Broadway a better choice? You will learn everything you need to know to answer these questions, start building highly concurrent applications in no time, and write code that’s not only fast, but also resilient to errors and easy to scale. + +
        + +

        Erlang in Angerfree

        + +Erlang in Anger cover + +This book intends to be a little guide about how to be the Erlang medic in a time of war. It is first and foremost a collection of tips and tricks to help understand where failures come from, and a dictionary of different code snippets and practices that helped developers debug production systems that were built in Erlang. + +
        + +## Courses + +

        Elixir Schoolfree

        + +Elixir School + +Elixir-School is an open and community driven effort inspired by Twitter's Scala School. The site's content consists of peer-reviewed lessons on various Elixir topics that range in difficulty. The lessons are currently available in over 10 languages to help make programming Elixir more accessible to non-English speakers. + +
        + +

        Pragmatic Studio's Elixir/OTP Course

        + +Pragmatic Studio's Elixir/OTP Course + +Put Elixir and OTP into action as you build a concurrent, fault-tolerant application from scratch in this 6-hour video course from The Pragmatic Studio. By developing a real app with real code, you'll gain practical experience putting all the pieces together to craft applications the Elixir/OTP way. + +The first half of the course focuses on core Elixir facets, principles, and techniques. In the second half, we go beyond the basics and focus on what sets Elixir/OTP apart: concurrent processes, the actor model, OTP behaviors, and fault recovery. + +If you're new to Elixir, you'll get step-by-step guidance in an engaging format you won't find anywhere else. If you have experience with Elixir, you'll gain a deeper understanding of things you've been taking for granted and fill in any knowledge gaps. + +
        + +

        grox.io's Elixir Course

        + +grox.io's Multi-Format Elixir Course + +Learning complex concepts like programming languages is best with multiple formats. Groxio's learning method embraces an interactive mini-book for beginners, video overviews for novices, an online book for presenting higher level concepts, videos with live coding to simulate advanced pair programming through projects meaningful to beginners and experts. + +The Elixir module is a full program with a book with 80 pages, 8 videos, dozens of exercises, and two full test-first projects. Beginners can learn the language, and experts can fill in typical blind spots like writing sigils, building macros, and using streams. + +
        + +

        grox.io's OTP Course

        + +grox.io's Multi-Format OTP Course + +This course teaches OTP from a design perspective by showing a system for breaking projects into layers. This course builds on those layering concepts with a 60 page book, 12 videos, projects, and curated links to go into deeper detail for tricky OTP concepts. + +Understand how back-pressure works, step inside supervision trees, and learn to build your dynamic supervisors. Learn OTP for the first time, or solidify your intuition by building the base concepts by reading a book, watching videos, and working through guided projects using Groxio's blend of media, designed to take you from novice to expert. + +
        + +

        ThinkingElixir.com's Pattern Matching Coursefree

        + +ThinkingElixir.com's Free Pattern Matching Course + +Pattern matching is a really powerful language feature. It is built in to almost every corner or Elixir. In order for you to even read Elixir code and follow along, you have to understand Pattern Matching. + +Once you "get" pattern matching, it feels like a super power. Pattern Matching makes new patterns of coding possible. You start to unlearn some of the patterns you've used in other languages because now you can create even clearer and more elegant code than was possible before! + +The course covers getting setup with Elixir, the data types, how to pattern match each of them, and more! An included TDD project helps you easily apply what you're learning. + +
        + +

        LearnElixir.tv

        + + + LearnElixir.tv cover + + +LearnElixir.tv is a video course which provides in-depth, step-by-step videos about Elixir's main features. Videos range from 7 to 15 minutes in length. + +It's intended to help beginners get familiar with all of Elixir's features by building their knowledge incrementally. Experienced Elixir developers might also learn a trick or two. + +
        + +

        Educative.io's Metaprogramming in Elixir Course

        + + + Educative.io's Metaprogramming in Elixir Course + + +Get introduced to the concept of metaprogramming. Learn how to level up your programming skills by discovering the full potential of the macro system in Elixir. Understand the ins and outs of metaprogramming at a fundamental level and write incredible libraries by doing more with less code. + +
        + +

        Learn-Elixir.dev

        + + + Learn-Elixir.dev cover + + +At your own pace, progress through 132 videos, 30 quizzes, 9 assignments, and 2 major projects of your choosing to demonstrate your working knowledge of a production-level quality of code. You’ll learn Syntax/Fundamentals, REST, GraphQL with Absinthe, Phoenix, OTP, Testing, Ecto, architecture, how to scale, how to go distributed, and more… + +Join our weekly live coaching sessions every Wednesday at 12:00-noon pacific time to close any gaps in knowledge and get specific answers to your specific questions! + +Found your start-up, migrate a codebase, build that app! Our midterm and final projects create a great win-win scenario for you to complete your personal project while mastering the language. Or if you’re more career-focused we can offer referrals, recommendations, or references and host your completed projects on GitHub to show hiring managers your proficiency programming Elixir. + +
        + +

        TechSchoolfree

        + + + TechSchool + + +TechSchool is an open-source platform that teaches programming through free YouTube videos and other websites. The goal is to make technology education accessible to everyone. It includes several Elixir courses and a complete Fullstack Elixir + Phoenix Bootcamp. + +
        + +## Screencasts + +

        ElixirStreamsfree

        + + + elixir streams cover + + +ElixirStreams provides free video tips (under 3 mins!) covering a variety of +Elixir and Phoenix topics. The videos help you sharpen the saw as you learn +about new tools and tricks, and they keep you up to date with the latest +developments in the language. + +
        + +

        ElixirCasts.iofree

        + + + elixircasts.io cover + + +ElixirCasts is a collection of simple screencasts that cover a wide range of Elixir and Phoenix topics. Each episode tackles a specific problem or explores a new library, demystifying it in a language that's easy to understand. + +Episodes range from beginner focused to more moderate and advanced topics. Come build your knowledge of Elixir with us, one episode at a time. + +
        + +

        Alchemist Campfree

        + +Alchemist Camp cover + +Alchemist Camp is the largest producer of free Elixir screencasts and has dozens of hours of screencasts on YouTube. The videos are often longer-form and focused around projects, such as building a small Phoenix clone, or an OTP worker to regularly collect statistics from multiple APIs. Content is driven by viewer request. + +Alchemist Camp is aimed at people who have some web development experience and want to ship real-world projects in Elixir. + +
        + +## Other resources + +

        Elixir Flashcards

        + +Elixir Flashcards + +Elixir flashcards are a set of beautifully crafted, professionally printed, poker sized flashcards to help you master the Elixir language. + +Flashcards are a great way to highlight knowledge gaps, identify misconceptions or false beliefs, and help you memorise key concepts. +When used in groups or teams, flashcards can help spark interesting discussions, and help bring people together to learn in a fun way by playing games. + +Combined with books, tutorials and screencasts, using flashcards is the killer combination to master Elixir. + +
        + +

        Elixir Koansfree

        + +Elixir Koans + +Elixir koans is a fun, easy way to get started with the Elixir programming language. It is an idiomatic tour of the language. + +
        + +

        Exercismfree

        + + Exercism Elixir track + +Exercism is an open source platform that provides free practice and mentoring in many languages, including Elixir. +It features exercises of varying difficulty, from string processing to using OTP, that are mentored by volunteers. +Once you have completed an exercise you can also view other students' solutions. + +
        + +

        Running in Production Podcastfree

        + +Running in Production Podcast + +Running in Production is a podcast where developers and engineers talk about +running small and large Elixir / Phoenix web apps in production. + +Topics include tech stacks, success stories, lessons learned and deployment tips. + +
        diff --git a/opensearch.xml b/opensearch.xml new file mode 100644 index 000000000..4e982e7c5 --- /dev/null +++ b/opensearch.xml @@ -0,0 +1,15 @@ + + + Search Elixir website + http://elixir-lang.org/ + http://elixir-lang.org/ + UTF-8 + en + + elixir-lang.org + + + http://elixir-lang.org/favicon.ico + \ No newline at end of file diff --git a/trademarks.markdown b/trademarks.markdown new file mode 100644 index 000000000..329585587 --- /dev/null +++ b/trademarks.markdown @@ -0,0 +1,70 @@ +--- +title: "Trademarks policy" +section: trademarks +layout: default +image: /images/social/elixir-og-card.jpg +--- + +# {{ page.title }} + +This document outlines the policy for allowed usage of the "Elixir" word and the Elixir logo by other parties. + +"Elixir" and the Elixir logo are registered trademarks of the Elixir Team. The Elixir Team believes in a decentralized approach to growing the community and the ecosystem, independent of the Elixir project and the Elixir Team. + +Anyone can use the Elixir trademarks if that use of the trademark is nominative. The trademarks must not be used to disparage the project and its community, nor be used in any way to imply ownership, endorsement, or association with the Elixir project and the Elixir Team. + +You must not visually combine the Elixir logo with any other images, or change the logo in any way other than ways required by printing restrictions. If you want to create your own visual identity in relation to Elixir, you might use the shape of an unrelated "water drop" as part of your design, as seen in many community projects and initiatives. You must not combine or modify the Elixir logo. + +The Elixir logo is [available in our repository](https://github.com/elixir-lang/elixir-lang.github.com/tree/main/downloads/logos) in both vertical and horizontal versions. + +## Nominative use + +The "nominative use" (or "nominative fair use") is a legal doctrine that authorizes everyone (even commercial companies) to use or refer to the trademark of another if: + + * The product or service in question must be one not readily identifiable without use of the trademark. + + * Only so much of the mark or marks may be used as is reasonably necessary to identify the product or service. + + * The organization using the mark must do nothing that would, in conjunction with the mark, suggest sponsorship or endorsement by the trademark holder. + +Our trademarks must be used to refer to the Elixir programming language. + +## Examples of permitted use + +All examples listed next must strictly adhere to the terms outlined in the previous sections: + + * Usage of the Elixir logo to say a technology is "powered by Elixir" under nominative use. Linking back to the Elixir website, if possible, is appreciated. + + * Usage of the Elixir logo to display it as a supported technology in a service or platform. For instance, you may say "we support Elixir" and use the Elixir logo, but you may not refer to yourself as "the Elixir platform" nor imply any form of endorsement or association with Elixir. + + * Usage of the Elixir logo in non-commercial community meetups, in presentations, and in courses when referring to the language and its ecosystem under nominative use. + + * Usage of the Elixir logo in non-commercial swag (stickers, t-shirts, mugs, etc) to promote the Elixir programming language. The Elixir marks must be the only marks featured in the product. You need permission to make swag that include Elixir and other third party marks in them. + + * Inclusion of the Elixir logo in non-commercial icon sets. Use of the Elixir icons must still adhere to Elixir's trademark policies. + + * Usage of the "Elixir" word in book titles, meetups, conferences, and podcasts. You must not use the word to imply uniqueness or endorsement from the Elixir team. "The Elixir book" and "The Elixir podcast" are not permitted. "Elixir in Action", "Thinking Elixir", and "Kraków Elixir User Group" are valid examples already in use today. + + * Usage of the "Elixir" word in the names of freely distributed software and hardware products is allowed when referring to use with or suitability for the Elixir programming language, such as wxElixir, Elixirsense, etc. If the product includes the Elixir programming language itself, then [you must also respect its license](https://github.com/elixir-lang/elixir/blob/main/LICENSE). + +## Examples of not permitted use + +Here is a non-exhaustive list of non permitted uses of the marks: + + * Usage of the Elixir logo in book covers, conferences, and podcasts. + + * Usage of the Elixir logo as the mark of third party projects, even in combination with other marks. + + * Naming any company or product after Elixir, such as "The Elixir Hosting", "The Elixir Consultants", etc. + +## Examples that require permission + +Here are some examples that may be granted permission upon request: + + * Selling merchandise (stickers, t-shirts, mugs, etc). + +You can request permission by emailing trademarks@elixir-lang.org. + +## Important note + +Nothing in this page shall be interpreted to allow any third party to claim any association with the Elixir project and the Elixir Team, or to imply any approval or support by the Elixir project and the Elixir Team for any third party products, services, or events.