Compare commits
16 Commits
stealthy_s
...
master
| Author | SHA1 | Date |
|---|---|---|
Rob Watson
|
696a71a193 | |
|
Rob Watson
|
5045f0a3d4 | |
|
Rob Watson
|
a8c4e5bafe | |
|
Rob Watson
|
b930441ba4 | |
|
Rob Watson
|
40b79f5382 | |
|
Rob Watson
|
b6c5e4c59d | |
|
Rob Watson
|
65eb0376ce | |
|
Rob Watson
|
5f79502d91 | |
|
Rob Watson
|
30a51fefa4 | |
|
Rob Watson
|
1d11401a1e | |
|
Rob Watson
|
96a6491043 | |
|
Rob Watson
|
2f3e9f0565 | |
|
Rob Watson
|
209a2ba957 | |
|
Rob Watson
|
05aef5fe9d | |
|
Rob Watson
|
fdbbc5e4c5 | |
|
Rob Watson
|
d88522dad0 |
|
|
@ -3,3 +3,5 @@ _production
|
||||||
.sass-cache
|
.sass-cache
|
||||||
.jekyll-metadata
|
.jekyll-metadata
|
||||||
.byebug_history
|
.byebug_history
|
||||||
|
*.backup
|
||||||
|
/NOTES.md
|
||||||
|
|
|
||||||
23
Gemfile
23
Gemfile
|
|
@ -1,28 +1,13 @@
|
||||||
source "https://rubygems.org"
|
source "https://rubygems.org"
|
||||||
|
|
||||||
# Hello! This is where you manage which Jekyll version is used to run.
|
gem "jekyll"
|
||||||
# When you want to use a different version, change it below, save the
|
gem "minima", "~> 2.0", git: 'https://github.com/rfwatson/minima-2col'
|
||||||
# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:
|
gem "pygments.rb"
|
||||||
#
|
|
||||||
# bundle exec jekyll serve
|
|
||||||
#
|
|
||||||
# This will help ensure the proper Jekyll version is running.
|
|
||||||
# Happy Jekylling!
|
|
||||||
gem "jekyll", "~> 3.6.2"
|
|
||||||
|
|
||||||
# This is the default theme for new Jekyll sites. You may change this to anything you like.
|
|
||||||
gem "minima", "~> 2.0", path: '/home/rob/dev/minima'
|
|
||||||
|
|
||||||
# If you want to use GitHub Pages, remove the "gem "jekyll"" above and
|
|
||||||
# uncomment the line below. To upgrade, run `bundle update github-pages`.
|
|
||||||
# gem "github-pages", group: :jekyll_plugins
|
|
||||||
|
|
||||||
# If you have any plugins, put them here!
|
|
||||||
group :jekyll_plugins do
|
group :jekyll_plugins do
|
||||||
gem "jekyll-feed", "~> 0.6"
|
gem "jekyll-feed", "~> 0.6"
|
||||||
gem 'jekyll-seo-tag'
|
gem 'jekyll-seo-tag'
|
||||||
gem 'jekyll-stealthy-share', path: '/home/rob/dev/jekyll-stealthy-share'
|
gem 'jekyll-stealthy-share', git: 'https://github.com/rfwatson/jekyll-stealthy-share'
|
||||||
end
|
end
|
||||||
|
|
||||||
# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
|
|
||||||
gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
|
gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
|
||||||
|
|
|
||||||
36
Gemfile.lock
36
Gemfile.lock
|
|
@ -1,10 +1,12 @@
|
||||||
PATH
|
GIT
|
||||||
remote: /home/rob/dev/jekyll-stealthy-share
|
remote: https://github.com/rfwatson/jekyll-stealthy-share
|
||||||
|
revision: 7215f5f41c728dca97c7e1762d3e36ed9342367a
|
||||||
specs:
|
specs:
|
||||||
jekyll-stealthy-share (0.1.0)
|
jekyll-stealthy-share (0.1.0)
|
||||||
|
|
||||||
PATH
|
GIT
|
||||||
remote: /home/rob/dev/minima
|
remote: https://github.com/rfwatson/minima-2col
|
||||||
|
revision: e67fdc20590db7d27555736bc6ad75314f3fd0dc
|
||||||
specs:
|
specs:
|
||||||
minima (2.1.1)
|
minima (2.1.1)
|
||||||
jekyll (~> 3.5)
|
jekyll (~> 3.5)
|
||||||
|
|
@ -16,18 +18,28 @@ GEM
|
||||||
addressable (2.5.2)
|
addressable (2.5.2)
|
||||||
public_suffix (>= 2.0.2, < 4.0)
|
public_suffix (>= 2.0.2, < 4.0)
|
||||||
colorator (1.1.0)
|
colorator (1.1.0)
|
||||||
|
concurrent-ruby (1.0.5)
|
||||||
|
em-websocket (0.5.1)
|
||||||
|
eventmachine (>= 0.12.9)
|
||||||
|
http_parser.rb (~> 0.6.0)
|
||||||
|
eventmachine (1.2.5)
|
||||||
ffi (1.9.18)
|
ffi (1.9.18)
|
||||||
forwardable-extended (2.6.0)
|
forwardable-extended (2.6.0)
|
||||||
jekyll (3.6.2)
|
http_parser.rb (0.6.0)
|
||||||
|
i18n (0.9.3)
|
||||||
|
concurrent-ruby (~> 1.0)
|
||||||
|
jekyll (3.7.2)
|
||||||
addressable (~> 2.4)
|
addressable (~> 2.4)
|
||||||
colorator (~> 1.0)
|
colorator (~> 1.0)
|
||||||
|
em-websocket (~> 0.5)
|
||||||
|
i18n (~> 0.7)
|
||||||
jekyll-sass-converter (~> 1.0)
|
jekyll-sass-converter (~> 1.0)
|
||||||
jekyll-watch (~> 1.1)
|
jekyll-watch (~> 2.0)
|
||||||
kramdown (~> 1.14)
|
kramdown (~> 1.14)
|
||||||
liquid (~> 4.0)
|
liquid (~> 4.0)
|
||||||
mercenary (~> 0.3.3)
|
mercenary (~> 0.3.3)
|
||||||
pathutil (~> 0.9)
|
pathutil (~> 0.9)
|
||||||
rouge (>= 1.7, < 3)
|
rouge (>= 1.7, < 4)
|
||||||
safe_yaml (~> 1.0)
|
safe_yaml (~> 1.0)
|
||||||
jekyll-feed (0.9.2)
|
jekyll-feed (0.9.2)
|
||||||
jekyll (~> 3.3)
|
jekyll (~> 3.3)
|
||||||
|
|
@ -35,7 +47,7 @@ GEM
|
||||||
sass (~> 3.4)
|
sass (~> 3.4)
|
||||||
jekyll-seo-tag (2.4.0)
|
jekyll-seo-tag (2.4.0)
|
||||||
jekyll (~> 3.3)
|
jekyll (~> 3.3)
|
||||||
jekyll-watch (1.5.1)
|
jekyll-watch (2.0.0)
|
||||||
listen (~> 3.0)
|
listen (~> 3.0)
|
||||||
kramdown (1.16.2)
|
kramdown (1.16.2)
|
||||||
liquid (4.0.0)
|
liquid (4.0.0)
|
||||||
|
|
@ -44,13 +56,16 @@ GEM
|
||||||
rb-inotify (~> 0.9, >= 0.9.7)
|
rb-inotify (~> 0.9, >= 0.9.7)
|
||||||
ruby_dep (~> 1.2)
|
ruby_dep (~> 1.2)
|
||||||
mercenary (0.3.6)
|
mercenary (0.3.6)
|
||||||
|
multi_json (1.13.1)
|
||||||
pathutil (0.16.1)
|
pathutil (0.16.1)
|
||||||
forwardable-extended (~> 2.6)
|
forwardable-extended (~> 2.6)
|
||||||
public_suffix (3.0.1)
|
public_suffix (3.0.1)
|
||||||
|
pygments.rb (1.2.1)
|
||||||
|
multi_json (>= 1.0.0)
|
||||||
rb-fsevent (0.10.2)
|
rb-fsevent (0.10.2)
|
||||||
rb-inotify (0.9.10)
|
rb-inotify (0.9.10)
|
||||||
ffi (>= 0.5.0, < 2)
|
ffi (>= 0.5.0, < 2)
|
||||||
rouge (2.2.1)
|
rouge (3.1.0)
|
||||||
ruby_dep (1.5.0)
|
ruby_dep (1.5.0)
|
||||||
safe_yaml (1.0.4)
|
safe_yaml (1.0.4)
|
||||||
sass (3.5.5)
|
sass (3.5.5)
|
||||||
|
|
@ -63,11 +78,12 @@ PLATFORMS
|
||||||
ruby
|
ruby
|
||||||
|
|
||||||
DEPENDENCIES
|
DEPENDENCIES
|
||||||
jekyll (~> 3.6.2)
|
jekyll
|
||||||
jekyll-feed (~> 0.6)
|
jekyll-feed (~> 0.6)
|
||||||
jekyll-seo-tag
|
jekyll-seo-tag
|
||||||
jekyll-stealthy-share!
|
jekyll-stealthy-share!
|
||||||
minima (~> 2.0)!
|
minima (~> 2.0)!
|
||||||
|
pygments.rb
|
||||||
tzinfo-data
|
tzinfo-data
|
||||||
|
|
||||||
BUNDLED WITH
|
BUNDLED WITH
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ permalink: /blog/:year/:month/:day/:title/
|
||||||
google_analytics: UA-8059487-21
|
google_analytics: UA-8059487-21
|
||||||
disqus:
|
disqus:
|
||||||
shortname: netflux
|
shortname: netflux
|
||||||
|
highlighter: pygments
|
||||||
|
|
||||||
# Build settings
|
# Build settings
|
||||||
markdown: kramdown
|
markdown: kramdown
|
||||||
|
|
@ -36,7 +37,8 @@ plugins:
|
||||||
# Exclude from processing.
|
# Exclude from processing.
|
||||||
# The following items will not be processed, by default. Create a custom list
|
# The following items will not be processed, by default. Create a custom list
|
||||||
# to override the default setting.
|
# to override the default setting.
|
||||||
# exclude:
|
exclude:
|
||||||
|
- '*.backup'
|
||||||
# - Gemfile
|
# - Gemfile
|
||||||
# - Gemfile.lock
|
# - Gemfile.lock
|
||||||
# - node_modules
|
# - node_modules
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
<img src="/assets/images/me.png" class="me"/>
|
<img src="/assets/images/me.png" class="me"/>
|
||||||
<p>Welcome to <strong>Netflux.io</strong>, the tech blog of Rob Watson.</p>
|
<p>Welcome to <strong>Netflux.io</strong>, the tech blog of Rob Watson.</p>
|
||||||
<p>I am co-founder and CTO of <a href="http://mixlr.com">Mixlr</a>.
|
|
||||||
<p>You can find me on <a href="https://twitter.com/rfwatson">Twitter</a>, <a href="https://github.com/rfwatson">GitHub</a>, <a href="https://gitlab.com/rfwatson">GitLab</a> and <a href="https://www.linkedin.com/in/rfw21">LinkedIn</a>.
|
<p>You can find me on <a href="https://twitter.com/rfwatson">Twitter</a>, <a href="https://github.com/rfwatson">GitHub</a>, <a href="https://gitlab.com/rfwatson">GitLab</a> and <a href="https://www.linkedin.com/in/rfw21">LinkedIn</a>.
|
||||||
</p>
|
</p>
|
||||||
<p><a href="https://twitter.com/rfwatson" target="_blank"><img src="/assets/images/follow.png"></a></p>
|
<p><a href="https://twitter.com/rfwatson" target="_blank"><img src="/assets/images/follow.png"></a></p>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
---
|
||||||
|
layout: post
|
||||||
|
title: "2017 roundup part 1: Thes gem"
|
||||||
|
slug: thes-gem
|
||||||
|
date: 2018-01-16 07:00:00 +0000
|
||||||
|
categories: ruby gems foss
|
||||||
|
---
|
||||||
|
|
||||||
|
When writing, I often need to refer to a thesaurus. But I find visiting the best online option, [www.thesaurus.com](http://www.thesaurus.com), to be an unpleasant and distracting experience.
|
||||||
|
|
||||||
|
So I decided to write a tool to improve matters. It's called `thes` and while it's a very simple utility, it's also one that I've found consistently useful and time-saving over the last few months.
|
||||||
|
|
||||||
|
The idea is simple: simply enter `thes <some search term>`, and the gem will search [www.thesaurus.com](http://www.thesaurus.com) and print the results straight back to the console in table format. Think of it as a command-line thesaurus client, with no bandwidth-wasting JavaScript, assets and adverts or annoyingly colourful design!
|
||||||
|
|
||||||
|
Here's an example of entering `thes incredible`:
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Thes is available as a gem:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
gem install thes
|
||||||
|
```
|
||||||
|
|
||||||
|
or, alternatively, on GitHub:
|
||||||
|
[https://github.com/rfwatson/thes](https://github.com/rfwatson/thes)
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
---
|
||||||
|
layout: post
|
||||||
|
title: "2017 roundup part 2: rack-filter-param"
|
||||||
|
slug: rack-filter-param
|
||||||
|
date: 2018-01-18 06:00:00 +0000
|
||||||
|
categories: ruby gems foss rack
|
||||||
|
---
|
||||||
|
|
||||||
|
Hot on the heels of [thes](/blog/2018/01/16/thes-gem/) is another Ruby gem: this time containing some [Rack](https://rack.github.io/) middleware.
|
||||||
|
|
||||||
|
[rack-filter-param](https://github.com/rfwatson/rack-filter-param/) solves a simple problem. As the name suggests, it allows HTTP parameters to be filtered from incoming requests based on arbitrary application logic.
|
||||||
|
|
||||||
|
The middleware needs only the most minimal configuration. For example, here it is being configured to strip the `client_id` parameter from incoming requests, based on some assumed application-level logic:
|
||||||
|
|
||||||
|
{% highlight ruby %}
|
||||||
|
use Rack::FilterParam, {
|
||||||
|
param: :client_id,
|
||||||
|
if: -> (value) { should_I_do_it?(value) }
|
||||||
|
}
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
When the application method `should_I_do_it?` returns a truthy value, the `client_id` parameter is stripped from the request, which is then passed upstream.
|
||||||
|
|
||||||
|
It is a library born of a real-world refactoring problem. A legacy authentication library, that had been part of Mixlr's API, was being replaced with a newer library. While doing so it was discovered that a particular HTTP querystring parameter, hard-coded into numerous client apps, triggered some unwanted and buggy behaviour in the new library.
|
||||||
|
|
||||||
|
Fixing the problem by pushing out updates to the client apps, and removing the parameter completely, wasn't an option. That would have left the app broken for all users who didn't or couldn't update to the new version. And patching the new library felt wrong too, because it would only increase future maintenance burden and make it more difficult to keep the library up-to-date.
|
||||||
|
|
||||||
|
The next step was to attempt to remove the parameter using Nginx, but the unwanted behaviour was only triggered when the parameter had certain values. In other words, the removal of the parameter depended on application logic, and application logic belongs in the application and not hidden in the front-end web server configuration.
|
||||||
|
|
||||||
|
So the correct solution appeared to be to manipulate the request at the Rack middleware layer, which is part of the application but kicks in early enough to be able to remove the parameter in good time. And extracting it as a gem kept the application itself clean of legacy code paths, and allowed it to be re-used by anybody in the future.
|
||||||
|
|
||||||
|
You can read more and browse the source code over on my GitHub page.
|
||||||
|
|
||||||
|
[https://github.com/rfwatson/rack-filter-param](https://github.com/rfwatson/rack-filter-param)
|
||||||
|
|
@ -0,0 +1,131 @@
|
||||||
|
---
|
||||||
|
layout: post
|
||||||
|
title: "2017 roundup part 3: routing_report"
|
||||||
|
slug: routing-report-gem
|
||||||
|
date: 2018-01-19 07:00:00 +0000
|
||||||
|
categories: ruby rails gems foss cruft
|
||||||
|
---
|
||||||
|
|
||||||
|
A tiresome but essential aspect of managing large-scale applications is keeping track of _cruft_ — [succinctly defined](https://en.wikipedia.org/wiki/Cruft) by Wikipedia as
|
||||||
|
|
||||||
|
> anything that is left over, redundant and getting in the way.
|
||||||
|
|
||||||
|
Ruby on Rails applications are naturally no exception to this principle. Let's take two of the most elemental components of a Rails app: routes and actions.
|
||||||
|
|
||||||
|
Rails will not explicitly warn you if a route is added that has no corresponding action, or vice versa. Instead it will simply allow your app to accumulate cruft. Although trying to access such an action or route will immediately throw a `RoutingError` or return a 404 response, this does not help to avoid the number of unnecessary routes and actions from increasing over time.
|
||||||
|
|
||||||
|
A disciplined approach to development and careful code review can help to avoid this fate, but the larger an application gets—and the more developers that work on it—it becomes ever more likely that superfluous routes or actions will be collected.
|
||||||
|
|
||||||
|
This is where another gem that I released in 2017, [routing_report](https://github.com/rfwatson/routing_report), comes in.
|
||||||
|
|
||||||
|
The gem adds a single Rake task to your Rails application. When you run the task, a report is generated that highlights both routes that have no reciprocal actions, and actions that have no corresponding routes.
|
||||||
|
|
||||||
|
To install the gem, just add it to your Gemfile:
|
||||||
|
|
||||||
|
{% highlight ruby %}
|
||||||
|
group :development do
|
||||||
|
gem 'routing_report'
|
||||||
|
end
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
And discover unwanted routes and actions by running the Rake task:
|
||||||
|
|
||||||
|
{% highlight bash %}
|
||||||
|
rake routing_report:run
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
Here is the report for the popular open source Rails app [GitLab](https://gitlab.com/gitlab-org/gitlab-ce):
|
||||||
|
|
||||||
|
```
|
||||||
|
+------------------------------------------------------------------------------+
|
||||||
|
| Routes without actions (18) |
|
||||||
|
+------------------------------------------------------------------------------+
|
||||||
|
| doorkeeper/token_info#show |
|
||||||
|
| doorkeeper/tokens#create |
|
||||||
|
| doorkeeper/tokens#revoke |
|
||||||
|
| oauth/applications#edit |
|
||||||
|
| oauth/applications#show |
|
||||||
|
| oauth/authorizations#show |
|
||||||
|
| projects/badges#index |
|
||||||
|
| projects/boards#create |
|
||||||
|
| projects/boards#destroy |
|
||||||
|
| projects/boards#update |
|
||||||
|
| projects/boards#update |
|
||||||
|
| projects/branches#new |
|
||||||
|
| projects/merge_requests#diff_for_path |
|
||||||
|
| projects/milestones#sort_issues |
|
||||||
|
| projects/milestones#sort_merge_requests |
|
||||||
|
| projects/services#index |
|
||||||
|
| projects/tags#new |
|
||||||
|
| snippets/notes#delete_attachment |
|
||||||
|
+------------------------------------------------------------------------------+
|
||||||
|
|
||||||
|
+------------------------------------------------------------------------------+
|
||||||
|
| Actions without routes (57) |
|
||||||
|
+------------------------------------------------------------------------------+
|
||||||
|
| application#not_found |
|
||||||
|
| application#redirect_back_or_default |
|
||||||
|
| devise#_prefixes |
|
||||||
|
| devise/confirmations#create |
|
||||||
|
| devise/confirmations#new |
|
||||||
|
| devise/confirmations#show |
|
||||||
|
| devise/omniauth_callbacks#failure |
|
||||||
|
| devise/omniauth_callbacks#passthru |
|
||||||
|
| devise/passwords#create |
|
||||||
|
| devise/passwords#edit |
|
||||||
|
| devise/passwords#new |
|
||||||
|
| devise/passwords#update |
|
||||||
|
| devise/registrations#cancel |
|
||||||
|
| devise/registrations#create |
|
||||||
|
| devise/registrations#destroy |
|
||||||
|
| devise/registrations#edit |
|
||||||
|
| devise/registrations#new |
|
||||||
|
| devise/registrations#update |
|
||||||
|
| devise/sessions#create |
|
||||||
|
| devise/sessions#destroy |
|
||||||
|
| devise/sessions#new |
|
||||||
|
| doorkeeper/applications#create |
|
||||||
|
| doorkeeper/applications#destroy |
|
||||||
|
| doorkeeper/applications#index |
|
||||||
|
| doorkeeper/applications#new |
|
||||||
|
| doorkeeper/applications#update |
|
||||||
|
| doorkeeper/authorizations#create |
|
||||||
|
| doorkeeper/authorizations#destroy |
|
||||||
|
| doorkeeper/authorizations#new |
|
||||||
|
| doorkeeper/authorized_applications#destroy |
|
||||||
|
| doorkeeper/authorized_applications#index |
|
||||||
|
| omniauth_callbacks#authentiq |
|
||||||
|
| omniauth_callbacks#cas3 |
|
||||||
|
| omniauth_callbacks#failure_message |
|
||||||
|
| omniauth_callbacks#ldap |
|
||||||
|
| omniauth_callbacks#saml |
|
||||||
|
| profiles/notifications#user_params |
|
||||||
|
| projects/branches#recent |
|
||||||
|
| projects/git_http_client#actor |
|
||||||
|
| projects/git_http_client#authenticated_user |
|
||||||
|
| projects/git_http_client#authentication_abilities |
|
||||||
|
| projects/git_http_client#authentication_result |
|
||||||
|
| projects/git_http_client#redirected_path |
|
||||||
|
| projects/git_http_client#user |
|
||||||
|
| projects/merge_requests/conflicts#authorize_can_resolve_conflicts! |
|
||||||
|
| projects/network#assign_commit |
|
||||||
|
| projects/protected_refs#create |
|
||||||
|
| projects/protected_refs#destroy |
|
||||||
|
| projects/protected_refs#index |
|
||||||
|
| projects/protected_refs#show |
|
||||||
|
| projects/protected_refs#update |
|
||||||
|
| sherlock/application#find_transaction |
|
||||||
|
| sherlock/file_samples#show |
|
||||||
|
| sherlock/queries#show |
|
||||||
|
| sherlock/transactions#destroy_all |
|
||||||
|
| sherlock/transactions#index |
|
||||||
|
| sherlock/transactions#show |
|
||||||
|
+------------------------------------------------------------------------------+
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
The results aren't foolproof yet—for example, gems like [Devise](https://github.com/plataformatec/devise) and [Doorkeeper](https://github.com/doorkeeper-gem/doorkeeper) that satisfy routes in unconventional ways tend to generate false positives. This would need to be improved before the gem could be extended to be used programatically, for example as part of an continuous integration flow. (Pull requests are welcome by the way!)
|
||||||
|
|
||||||
|
But for manually highlighting redundant actions and dispensable routes the gem is already a useful tool.
|
||||||
|
|
||||||
|
You can find it on GitHub here: [https://github.com/rfwatson/routing_report](https://github.com/rfwatson/routing_report).
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
---
|
||||||
|
layout: post
|
||||||
|
title: "Using pg_isready to optimize test runs"
|
||||||
|
slug: pg_isready
|
||||||
|
date: 2018-01-23 13:00:00 +0000
|
||||||
|
categories: postgresql bash
|
||||||
|
---
|
||||||
|
|
||||||
|
While working with Docker Compose to set up a containerized RSpec testing environment today, I ran into a problem.
|
||||||
|
|
||||||
|
RSpec, running in one container, was intermittently trying to access the PostgreSQL database, located in a second container, before it had finished booting. Naturally, this was causing sporadic connection errors and test failures.
|
||||||
|
|
||||||
|
The testing environment was distributed, with many discrete Docker Compose applications running in parallel. And PostgreSQL's launch time varied considerably, sometimes being under a couple of seconds, but occasionally taking ten seconds or more.
|
||||||
|
|
||||||
|
This meant that naively setting a timeout (`sleep 5 && bundle exec rspec`...) was an unsatisfactory approach. It would have increased the run time of every test environment, but without completely solving the problem.
|
||||||
|
|
||||||
|
While researching the most appropriate solution, I discovered a useful tool. [`pg_isready`](https://www.postgresql.org/docs/9.3/static/app-pg-isready.html) allows a PostgreSQL database's status to be queried quickly and simply from a script or the command line.
|
||||||
|
|
||||||
|
Once I'd found it, using it in the RSpec container's entrypoint script was simple:
|
||||||
|
|
||||||
|
{% highlight bash %}
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
echo "Launching Rails $RAILS_ENV environment with target: $@"
|
||||||
|
|
||||||
|
until pg_isready -q -d my_database_name
|
||||||
|
do
|
||||||
|
echo "Waiting for database to be ready..."
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
bundle exec rake db:reset
|
||||||
|
bundle exec rspec $@
|
||||||
|
{% endhighlight %}
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
---
|
||||||
|
title: Launching Fargate instances from AWS Lambda
|
||||||
|
slug: aws-lambda-ecs-runtask
|
||||||
|
layout: post
|
||||||
|
categories: aws serverless
|
||||||
|
date: '2018-01-30 01:00:00 +0000'
|
||||||
|
---
|
||||||
|
|
||||||
|
Over the last few days, I've been experimenting with serverless web application development. This has included testing out [Fargate](https://aws.amazon.com/fargate/), Amazon's new managed container deployment service, and the more established [Lambda](https://aws.amazon.com/lambda/) and [API Gateway](https://aws.amazon.com/api-gateway/) services.
|
||||||
|
|
||||||
|
The end result that I've been trying to achieve is to use Lambda to launch, via an API Gateway endpoint, a one-off asynchronous container execution on Fargate. So far, I've managed to put most of the jigsaw pieces together with only one major blocking experience.
|
||||||
|
|
||||||
|
The problem was giving the Lambda execution role the requisite permissions to launch ECS instances (which can include Fargate instances) automatically. There are a couple of blog posts on the subject out there, and [one in particular](https://lobster1234.github.io/2017/12/03/run-tasks-with-aws-fargate-and-lambda/) that states that the Lambda role needs two policies: one that allows the `ecs:RunTask` action on the relevant resources, and another that adds the `iam:PassRole` that allows the `ecs:RunTask` role to be passed onto the task execution service itself.
|
||||||
|
|
||||||
|
This seems reasonably clear, but several hours of trying to make the [Visual Editor](https://aws.amazon.com/blogs/security/use-the-new-visual-editor-to-create-and-modify-your-aws-iam-policies/) apply the policies correctly I was still receiving permissions errors when Lambda tried to launch the Fargate container.
|
||||||
|
|
||||||
|
```
|
||||||
|
User: arn:aws:sts::123456789012:assumed-role/my-lambda-func-role/myFunc is not authorized to perform: iam:PassRole on resource: arn:aws:iam::123456789012:role/ecsTaskExecutionRole
|
||||||
|
```
|
||||||
|
|
||||||
|
The solution ended up to be very simple. Switching to the JSON view in the Visual Editor showed that the generated JSON was pretty far away from what I was expecting it to be. Directly editing the JSON, and pasting in the following policies, resolved the issue immediately.
|
||||||
|
|
||||||
|
{% highlight json %}
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Sid": "VisualEditor0",
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"ecs:RunTask"
|
||||||
|
],
|
||||||
|
"Resource": [
|
||||||
|
"*"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Sid": "VisualEditor1",
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"iam:PassRole"
|
||||||
|
],
|
||||||
|
"Resource": [
|
||||||
|
"arn:aws:iam::123456789012:role/ecsTaskExecutionRole"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
{% endhighlight %}
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
---
|
||||||
|
title: Adding a Jekyll helper script
|
||||||
|
slug: generate-new-jekyll-post
|
||||||
|
layout: post
|
||||||
|
categories: ''
|
||||||
|
force: false
|
||||||
|
date: '2018-01-30 00:00:00 +0000'
|
||||||
|
---
|
||||||
|
|
||||||
|
After a month or so of blogging with [Jekyll](https://jekyllrb.com/), I'm very happy with the platform. In particular, I feel that having dived much deeper into its internal workings than I had done before, I expect it to be able to support my pretty basic blogging needs well into the future.
|
||||||
|
|
||||||
|
One minor annoyance has been the lack of an easy way to generate new post templates. This is an irritation because having to manually create a new file and manually fill in all the [front matter](https://jekyllrb.com/docs/frontmatter/) values adds friction to a process that should be as fluid as possible.
|
||||||
|
|
||||||
|
So today I've added a simple but useful script to my Jenkins repo, which allows a new post to be generated from the command line with minimal effort.
|
||||||
|
|
||||||
|
It provides the following interface:
|
||||||
|
|
||||||
|
```
|
||||||
|
Usage: new [options]
|
||||||
|
-t, --title TITLE Title for post
|
||||||
|
-s, --slug SLUG Slug for post
|
||||||
|
-l, --layout LAYOUT Layout for post
|
||||||
|
-c, --category CATEGORY Add category for post (can be specified multiple times)
|
||||||
|
-f, --force Overwrite existing file
|
||||||
|
```
|
||||||
|
|
||||||
|
So this command:
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/new -t "Here's a new post" -s new-post-here -c tutorials -c random
|
||||||
|
```
|
||||||
|
|
||||||
|
generates a new post with an appropriate date and the following front matter:
|
||||||
|
|
||||||
|
```
|
||||||
|
---
|
||||||
|
title: Here's a new post
|
||||||
|
slug: new-post-here
|
||||||
|
layout: post
|
||||||
|
categories: tutorials random
|
||||||
|
date: '2018-01-30 00:00:00 +0000'
|
||||||
|
---
|
||||||
|
|
||||||
|
Hello reader...
|
||||||
|
```
|
||||||
|
|
||||||
|
I expect that this will make it even easier to post on this blog! If you could find this useful, it's available [on GitHub](https://github.com/rfwatson/techblog/blob/master/bin/new).
|
||||||
|
|
@ -0,0 +1,140 @@
|
||||||
|
---
|
||||||
|
title: Some things that I learned making GitLab Pages HTTPS-only
|
||||||
|
slug: things-I-learnt-gitlab-pages-https-only
|
||||||
|
layout: post
|
||||||
|
categories: 'development foss'
|
||||||
|
date: '2018-03-25 00:00:00 +0200'
|
||||||
|
---
|
||||||
|
|
||||||
|
This week, my most significant contribution yet to the [GitLab project](https://gitlab.com/) was [merged](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/16273) into master.
|
||||||
|
|
||||||
|
The contribution adds the ability for [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/index.html) websites to be served only over HTTPS connections, with plain HTTP requests 301-redirected to their secure counterpart.
|
||||||
|
|
||||||
|
The changes required to make this work were not trivial, and actually required two separate patches to be merged: one in [Ruby](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/16273) and a second in [Go](https://gitlab.com/gitlab-org/gitlab-pages/merge_requests/50).
|
||||||
|
|
||||||
|
Here's a random sampling of some things that I learnt during the development process.
|
||||||
|
|
||||||
|
### GitLab is a fun project to hack on
|
||||||
|
|
||||||
|
The [GitLab application](http://gitlab.com/gitlab-org/gitlab-ce/) consists of a Rails monolith plus a number of secondary services, including a dedicated [HTTP server for GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/) which is written in Go.
|
||||||
|
|
||||||
|
GitLab Pages websites are typically auto-generated by the GitLab [CI/CD system](https://about.gitlab.com/features/gitlab-ci-cd/). The artifacts of this process are static website assets, such as HTML files, which are served by the Pages server.
|
||||||
|
|
||||||
|
Implementing HTTPS-only pages required two separate changes to the overall application.
|
||||||
|
|
||||||
|
Firstly, the Rails monolith had to be altered to write [additional metadata](https://gitlab.com/gitlab-org/gitlab-ce/blob/9d45951fcaeda4f01a2e4be2480d980a3e7cd37e/app/services/projects/update_pages_configuration_service.rb#L19-35) to a per-project configuration file, indicating whether the project's Pages website should be forced to HTTPS or not. Supporting this were the user interface changes to allow a user to enable or disable the behaviour, and the usual layers of automated testing.
|
||||||
|
|
||||||
|
Secondly, the Go HTTP server had to be updated to parse the newly-added configuration, and adapt its behaviour accordingly (that is, serve a 301-redirect when appropriate).
|
||||||
|
|
||||||
|
I love the challenge of implementing a distributed feature across multiple technical domains. Being forced to consider the interaction between the services, error-handling, reliability and so on means that it's possible to learn much more than the sum of the two individual pull requests.
|
||||||
|
|
||||||
|
Additionally, GitLab's code quality in general feels very good. The Rails app is written in a modern style, with heavy use of [service objects](https://hackernoon.com/service-objects-in-ruby-on-rails-and-you-79ca8a1c946e) and comprehensive automated testing, including a very slick parallelized CI flow.
|
||||||
|
|
||||||
|
Finally, the GitLab community appears to be very professional and friendly, and contributing a significant feature was a very positive experience. Thanks guys!
|
||||||
|
|
||||||
|
### Go is a nice language
|
||||||
|
|
||||||
|
Generally I distrust Google as a corporation, and this political bias had led me to be somewhat suspicious about Go as a language. This turns out to have been a fallacy, and it's been great to dispel it.
|
||||||
|
|
||||||
|
Go is a much nicer language than I unfairly expected it to be. In its syntax, basic types and use of pointers it immediately reminded me of C, which made me feel comfortable very quickly.
|
||||||
|
|
||||||
|
I also found the Go toolchain to be very productive and enjoyable (although given Google's investment in the ultra-slick Android toolchain this probably shouldn't be a surprise). Being able to import modules directly from Git repositories, and auto-format code from the command-line are just two simple examples.
|
||||||
|
|
||||||
|
This project helped me to understand why Go has become so popular, and when it might be considered the right tool to choose in the future.
|
||||||
|
|
||||||
|
### Parameterized RSpec tests are a great thing
|
||||||
|
|
||||||
|
When writing RSpec tests, it is common to start with this kind of pattern:
|
||||||
|
|
||||||
|
{% highlight ruby %}
|
||||||
|
RSpec.describe "something" do
|
||||||
|
before do
|
||||||
|
send_request(path, "GET")
|
||||||
|
end
|
||||||
|
|
||||||
|
context "path is /" do
|
||||||
|
let(:path) { "/" }
|
||||||
|
|
||||||
|
it { is_successful }
|
||||||
|
end
|
||||||
|
|
||||||
|
context "path is /abc" do
|
||||||
|
let(:path) { "/abc" }
|
||||||
|
|
||||||
|
it { is_successful }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
This feels simple and reasonably readable, until a second contextual variable is introduced. Not only do we quickly see code-repetition being introduced, but the nesting also starts to feel unwieldy:
|
||||||
|
|
||||||
|
{% highlight ruby %}
|
||||||
|
RSpec.describe "something" do
|
||||||
|
before do
|
||||||
|
send_request(path, method)
|
||||||
|
end
|
||||||
|
|
||||||
|
context "path is /" do
|
||||||
|
let(:path) { "/" }
|
||||||
|
|
||||||
|
context "method is GET" do
|
||||||
|
let(:method) { "GET" }
|
||||||
|
it { is_successful }
|
||||||
|
end
|
||||||
|
|
||||||
|
context "method is POST" do
|
||||||
|
let(:method) { "POST" }
|
||||||
|
it { is_not_successful }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context "path is /abc" do
|
||||||
|
let(:path) { "/abc" }
|
||||||
|
|
||||||
|
context "method is GET" do
|
||||||
|
let(:method) { "GET" }
|
||||||
|
it { is_successful }
|
||||||
|
end
|
||||||
|
|
||||||
|
context "method is POST" do
|
||||||
|
let(:method) { "POST" }
|
||||||
|
it { is_not_successful }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
Introducing even more contextual levels exacerbates the problem further.
|
||||||
|
|
||||||
|
Enter [RSpec::Parameterzied](https://github.com/tomykaira/rspec-parameterized), which allows the same examples to be defined using a simple, flat table instead:
|
||||||
|
|
||||||
|
{% highlight ruby %}
|
||||||
|
RSpec.describe "something" do
|
||||||
|
using RSpec::Parameterized::TableSyntax
|
||||||
|
|
||||||
|
where(:path, :method, :success) do
|
||||||
|
"/" | "GET" | true
|
||||||
|
"/abc" | "GET" | true
|
||||||
|
"/" | "POST" | false
|
||||||
|
"/abc" | "POST" | false
|
||||||
|
end
|
||||||
|
|
||||||
|
with_them do
|
||||||
|
before do
|
||||||
|
send_request(path, "GET")
|
||||||
|
end
|
||||||
|
|
||||||
|
it "returns the expected response" do
|
||||||
|
expect(response.success).to eq success
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
RSpec will now run each of the rows in the table as a separate example, with inputs corresponding to the column values.
|
||||||
|
|
||||||
|
Although a downside of this approach is that debugging the specs can be more challenging, in test-heavy applications with a lot of contextual variants this feels like a useful option.
|
||||||
|
|
||||||
|
### Conclusions
|
||||||
|
|
||||||
|
Implementing HTTPS-only GitLab Pages has been a great learning experience, and a lot of fun! I'd highly recommend anybody who wants to hack on a non-trivial, backend-centric Rails app to check out the [GitLab Contributing page](https://about.gitlab.com/contributing/) and get involved.
|
||||||
2
about.md
2
about.md
|
|
@ -9,7 +9,7 @@ permalink: /about/
|
||||||
|
|
||||||
Hi, I'm Rob Watson and this is my personal tech blog.
|
Hi, I'm Rob Watson and this is my personal tech blog.
|
||||||
|
|
||||||
I co-founded live audio broadcasting company [Mixlr](http://mixlr.com), and have served as its CTO since 2011.
|
I co-founded live audio broadcasting company [Mixlr](http://mixlr.com), and served as its CTO from 2010 to 2018.
|
||||||
|
|
||||||
I also have a degree in [Music Informatics](https://en.wikipedia.org/wiki/Music_informatics) and have a long-standing interest in musical composition and performance using computers.
|
I also have a degree in [Music Informatics](https://en.wikipedia.org/wiki/Music_informatics) and have a long-standing interest in musical composition and performance using computers.
|
||||||
|
|
||||||
|
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 939 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 58 KiB |
|
|
@ -62,12 +62,13 @@ body > header.site-header {
|
||||||
}
|
}
|
||||||
|
|
||||||
.page-content {
|
.page-content {
|
||||||
background: #fff;
|
background: #f6f7f4;
|
||||||
padding-top: 0px;
|
padding-top: 0px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.outer-wrapper {
|
.outer-wrapper {
|
||||||
box-shadow: 5px 2px 15px -4px rgba(0, 0, 0, 0.1);
|
box-shadow: 5px 2px 15px -4px rgba(0, 0, 0, 0.1);
|
||||||
|
min-height: 700px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.inner-wrapper {
|
.inner-wrapper {
|
||||||
|
|
@ -93,8 +94,9 @@ img.profile {
|
||||||
|
|
||||||
.sidebar {
|
.sidebar {
|
||||||
line-height: 1.3em;
|
line-height: 1.3em;
|
||||||
background: #ddd;
|
background: linear-gradient(#eeefed, #fff);
|
||||||
text-align: center;
|
text-align: center;
|
||||||
|
overflow: auto;
|
||||||
|
|
||||||
img.me {
|
img.me {
|
||||||
width: 140px;
|
width: 140px;
|
||||||
|
|
@ -114,6 +116,7 @@ img.profile {
|
||||||
font-weight: 400;
|
font-weight: 400;
|
||||||
font-size: 18px;
|
font-size: 18px;
|
||||||
line-height: 1.5em;
|
line-height: 1.5em;
|
||||||
|
letter-spacing: 0.06px;
|
||||||
|
|
||||||
strong {
|
strong {
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
|
|
@ -140,3 +143,4 @@ img.profile {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
require "optparse"
|
||||||
|
require "time"
|
||||||
|
require "yaml"
|
||||||
|
|
||||||
|
front_matter = {
|
||||||
|
"title" => "Hello World",
|
||||||
|
"slug" => "hello-world",
|
||||||
|
"layout" => "post",
|
||||||
|
"categories" => []
|
||||||
|
}
|
||||||
|
|
||||||
|
options = {
|
||||||
|
force: false
|
||||||
|
}
|
||||||
|
|
||||||
|
OptionParser.new do |opts|
|
||||||
|
opts.on("-t", "--title TITLE", "Title for post") do |title|
|
||||||
|
front_matter["title"] = title
|
||||||
|
end
|
||||||
|
|
||||||
|
opts.on("-s", "--slug SLUG", "Slug for post") do |slug|
|
||||||
|
front_matter["slug"] = slug
|
||||||
|
end
|
||||||
|
|
||||||
|
opts.on("-l", "--layout LAYOUT", "Layout for post") do |layout|
|
||||||
|
front_matter["layout"] = layout
|
||||||
|
end
|
||||||
|
|
||||||
|
opts.on("-c", "--category CATEGORY", "Add category for post (can be specified multiple times)") do |category|
|
||||||
|
front_matter["categories"] << category
|
||||||
|
end
|
||||||
|
|
||||||
|
opts.on("-f", "--force", "Overwrite existing file") do |val|
|
||||||
|
options[:force] = val
|
||||||
|
end
|
||||||
|
end.parse!
|
||||||
|
|
||||||
|
now = DateTime.now
|
||||||
|
date = DateTime.new(now.year, now.month, now.day, 0, 0, 0, now.zone)
|
||||||
|
|
||||||
|
front_matter["date"] = date.to_time.to_s
|
||||||
|
front_matter["categories"] = front_matter["categories"].join(" ")
|
||||||
|
|
||||||
|
fname = date.strftime("_posts/%Y-%m-%d-#{front_matter["slug"]}.md")
|
||||||
|
raise "Already exists: #{fname}" if File.exists?(fname) && !options[:force]
|
||||||
|
|
||||||
|
File.open(fname, "w") do |f|
|
||||||
|
f << front_matter.to_yaml
|
||||||
|
f << "---" << "\n" << "\n"
|
||||||
|
f << "Hello reader..."
|
||||||
|
end
|
||||||
|
|
||||||
|
puts "Generated new post in: #{fname}"
|
||||||
Loading…
Reference in New Issue