Merge pull request #126 from fly-apps/125-add-litestream-as-an-option

Add --litestream as an option

README.md

@@ -67,6 +67,7 @@ are actually using.  But should you be using DATABASE_URL, for example, at runti
 additional support may be needed:
 
 * `--litefs` - use [LiteFS](https://fly.io/docs/litefs/)
+* `--litestream` - use [LiteFS](https://litestream.io/)
 * `--mysql` - add mysql libraries
 * `--postgresql` - add postgresql libraries
 * `--redis` - add redis libraries

lib/generators/dockerfile_generator.rb

@@ -2,6 +2,7 @@
 
 require "erb"
 require "json"
+require "shellwords"
 require_relative "../dockerfile-rails/scanner.rb"
 
 class DockerfileGenerator < Rails::Generators::Base
@@ -19,6 +20,7 @@ class DockerfileGenerator < Rails::Generators::Base
     "label" => {},
     "link" => false,
     "litefs" => false,
+    "litestream" => false,
     "lock" => true,
     "max-idle" => nil,
     "migrate" => "",
@@ -162,6 +164,9 @@ class DockerfileGenerator < Rails::Generators::Base
   class_option :litefs, type: :boolean, default: OPTION_DEFAULTS.litefs,
     desc: "replicate sqlite3 databases using litefs"
 
+  class_option :litestream, type: :boolean, default: OPTION_DEFAULTS.litestream,
+    desc: "replicate sqlite3 databases using litesream"
+
   class_option :tigris, type: :boolean, default: OPTION_DEFAULTS.tigris,
     desc: "configure active storage to use tigris"
 
@@ -346,6 +351,10 @@ def generate_app
       fly_attach_consul
     end
 
+    if using_litestream?
+      template "litestream.rake.erb", "lib/tasks/litestream.rake"
+    end
+
     if File.exist?("fly.toml") && (fly_processes || !options.prepare || options.swap || deploy_database == "sqlite3")
       if File.stat("fly.toml").size > 0
         template "fly.toml.erb", "fly.toml"
@@ -488,6 +497,10 @@ def using_litestack?
     @gemfile.include?("litestack")
   end
 
+  def using_litestream?
+    options.litestream?
+  end
+
   def using_node?
     return @using_node if @using_node != nil
     return if using_bun?
@@ -594,6 +607,10 @@ def install_gems
       system "bundle add mysql2 --skip-install" unless has_mysql_gem?
     end
 
+    if options.litestream?
+      system "bundle add litestream --skip-install" unless @gemfile.include? "litestream"
+    end
+
     if options.redis? || using_redis?
       system "bundle add redis --skip-install" unless @gemfile.include? "redis"
     end
@@ -1133,6 +1150,30 @@ def deploy_database
     end
   end
 
+  def start_command
+    if !options.procfile.blank?
+      ["foreman", "start", "--procfile=#{options.procfile}"]
+    elsif procfile.size > 1
+      ["foreman", "start", "--procfile=Procfile.prod"]
+    else
+      command = Shellwords.split(procfile.values.first)
+
+      if command.first == "./bin/thrust"
+        command[1..]
+      else
+        command
+      end
+    end
+  end
+
+  def shellescape(string)
+    if %r{\A[-\w._/= ]+\z}.match?(string)
+      string.inspect
+    else
+      Shellwords.escape(string)
+    end
+  end
+
   def node_version
     return unless using_node? || using_execjs?
 
@@ -1252,6 +1293,10 @@ def procfile
       }
     end
 
+    if using_litestream? && base[:rails]
+      base[:rails] = "./bin/rake litestream:run " + base[:rails]
+    end
+
     if solidq_launcher == :procfile
       base["solidq"] = "bundle exec rake solid_queue:start"
     end
@@ -1468,6 +1513,10 @@ def fly_make_toml
         list[primary] = list[primary].sub(/^.*thrust /, "")
       end
 
+      if using_litestream? && list["app"]
+        list["app"] = "./script/litestream #{list["app"]}"
+      end
+
       if toml.include? "[processes]"
         toml.sub!(/\[processes\].*?(\n\n|\n?\z)/m, "[processes]\n" +
           list.map { |name, cmd| "  #{name} = #{cmd.inspect}" }.join("\n") + '\1')

lib/generators/templates/docker-entrypoint.erb

@@ -36,17 +36,13 @@ fi
 <% end -%>
 <% if options.prepare -%>
 <% if !options.procfile.blank? -%>
-if [ "${*}" == "foreman start --procfile=<%= options.procfile %>" ]; then
+# If running the specified procfile then create or migrate existing database
 <% elsif procfile.size > 1 -%>
 # If running the production procfile then create or migrate existing database
-if [ "${*}" == "foreman start --procfile=Procfile.prod" ]; then
-<% elsif procfile.values.first.start_with? "./bin/rails server" -%>
-# If running the rails server then create or migrate existing database
-if [ "${1}" == "./bin/rails" ] && [ "${2}" == "server" ]<% if using_litefs? %> && [ "$FLY_REGION" == "$PRIMARY_REGION" ]<%end%>; then
 <% else -%>
 # If running the rails server then create or migrate existing database
-if [ "${*}" == <%= procfile.values.first.inspect %> <% if using_litefs? %>-a "$FLY_REGION" == "$PRIMARY_REGION" <%end%>]; then
 <% end -%>
+if <%= start_command.map.with_index {|word, index| "[ \"${@: #{index - start_command.length}:1}\" == #{shellescape(word)} ]"}.join(" && ") %><% if using_litefs? %> && [ "$FLY_REGION" == "$PRIMARY_REGION" ]<%end%>; then
 <% if options.precompile == "defer" -%>
   ./bin/rails assets:precompile
 <% end -%>

lib/generators/templates/litestream.rake.erb

@@ -0,0 +1,55 @@
+LITESTREAM_CONFIG = ENV["LITESTREAM_CONFIG"] || Rails.root.join("tmp/litestream.yml").to_s
+
+LITESTREAM_TEMPLATE = <<-EOF
+# This is the configuration file for litestream.
+#
+# For more details, see: https://litestream.io/reference/config/
+#
+dbs:
+<%% for db in @dbs -%>
+  - path: <%%= db %>
+    replicas:
+      - type: s3
+        endpoint: $AWS_ENDPOINT_URL_S3
+        bucket: $BUCKET_NAME
+        path: storage/<%%= File.basename(db) %>
+        access-key-id: $AWS_ACCESS_KEY_ID
+        secret-access-key: $AWS_SECRET_ACCESS_KEY
+<%% end -%>
+EOF
+
+namespace :litestream do
+  task prepare: "db:load_config" do
+    require "erubi"
+
+    @dbs =
+      ActiveRecord::Base
+        .configurations
+        .configs_for(env_name: "production", include_hidden: true)
+        .select { |config| [ "sqlite3", "litedb" ].include? config.adapter }
+        .map(&:database)
+
+    result = eval(Erubi::Engine.new(LITESTREAM_TEMPLATE).src)
+
+    unless File.exist?(LITESTREAM_CONFIG) && File.read(LITESTREAM_CONFIG) == result
+      File.write(LITESTREAM_CONFIG, result)
+    end
+
+    @dbs.each do |db|
+      next if File.exist?(db) or !ENV["BUCKET_NAME"]
+      system "litestream restore -config #{LITESTREAM_CONFIG} -if-replica-exists #{db}"
+      exit $?.exitstatus unless $?.exitstatus == 0
+    end
+  end
+
+  task :run do
+    require "shellwords"
+
+    exec(*%w[bundle exec litestream replicate -config],
+      LITESTREAM_CONFIG, "-exec", Shellwords.join(ARGV[1..-1]))
+  end
+end
+
+namespace :db do
+  task prepare: "litestream:prepare"
+end

test/results/alpine/docker-entrypoint

@@ -1,7 +1,7 @@
 #!/bin/sh -e
 
 # If running the rails server then create or migrate existing database
-if [ "${*}" == "./bin/thrust ./bin/rails server" ]; then
+if [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ]; then
   ./bin/rails db:prepare
 fi
 

test/results/idle/docker-entrypoint

@@ -9,7 +9,7 @@ swapon /swapfile
 echo 1 > /proc/sys/vm/overcommit_memory
 
 # If running the rails server then create or migrate existing database
-if [ "${*}" == "nginx" ]; then
+if [ "${@: -1:1}" == "nginx" ]; then
   ./bin/rails db:prepare
 fi
 

test/results/litefs/docker-entrypoint

@@ -4,7 +4,7 @@
 sudo -E litefs mount &
 
 # If running the rails server then create or migrate existing database
-if [ "${*}" == "./bin/thrust ./bin/rails server" -a "$FLY_REGION" == "$PRIMARY_REGION" ]; then
+if [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ] && [ "$FLY_REGION" == "$PRIMARY_REGION" ]; then
   ./bin/rails db:prepare
 fi
 

test/results/litestream/Dockerfile

@@ -0,0 +1,72 @@
+# syntax = docker/dockerfile:1
+
+# Make sure RUBY_VERSION matches the Ruby version in .ruby-version and Gemfile
+ARG RUBY_VERSION=xxx
+FROM ruby:$RUBY_VERSION-slim AS base
+
+# Rails app lives here
+WORKDIR /rails
+
+# Set production environment
+ENV BUNDLE_DEPLOYMENT="1" \
+    BUNDLE_PATH="/usr/local/bundle" \
+    BUNDLE_WITHOUT="development:test" \
+    RAILS_ENV="production"
+
+# Update gems and bundler
+RUN gem update --system --no-document && \
+    gem install -N bundler
+
+
+# Throw-away build stage to reduce size of final image
+FROM base AS build
+
+# Install packages needed to build gems
+RUN apt-get update -qq && \
+    apt-get install --no-install-recommends -y build-essential pkg-config
+
+# Install application gems
+COPY Gemfile Gemfile.lock ./
+RUN bundle install && \
+    bundle exec bootsnap precompile --gemfile && \
+    rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git
+
+# Copy application code
+COPY . .
+
+# Precompile bootsnap code for faster boot times
+RUN bundle exec bootsnap precompile app/ lib/
+
+# Precompiling assets for production without requiring secret RAILS_MASTER_KEY
+RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile
+
+
+# Final stage for app image
+FROM base
+
+# Install packages needed for deployment
+RUN apt-get update -qq && \
+    apt-get install --no-install-recommends -y curl libsqlite3-0 && \
+    rm -rf /var/lib/apt/lists /var/cache/apt/archives
+
+# Copy built artifacts: gems, application
+COPY --from=build "${BUNDLE_PATH}" "${BUNDLE_PATH}"
+COPY --from=build /rails /rails
+
+# Run and own only the runtime files as a non-root user for security
+RUN groupadd --system --gid 1000 rails && \
+    useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \
+    mkdir /data && \
+    chown -R 1000:1000 db log storage tmp /data
+USER 1000:1000
+
+# Deployment options
+ENV DATABASE_URL="sqlite3:///data/production.sqlite3"
+
+# Entrypoint prepares the database.
+ENTRYPOINT ["/rails/bin/docker-entrypoint"]
+
+# Start the server by default, this can be overwritten at runtime
+EXPOSE 80
+VOLUME /data
+CMD ["./bin/rake", "litestream:run", "./bin/thrust", "./bin/rails", "server"]

test/results/litestream/docker-entrypoint

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+# If running the rails server then create or migrate existing database
+if [ "${@: -5:1}" == "./bin/rake" ] && [ "${@: -4:1}" == litestream:run ] && [ "${@: -3:1}" == "./bin/thrust" ] && [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ]; then
+  ./bin/rails db:prepare
+fi
+
+exec "${@}"

test/results/litestream/fly.toml

@@ -0,0 +1,14 @@
+
+[env]
+  PORT = "8080"
+
+[processes]
+  app = "./script/litestream ./bin/rails server"
+
+[mounts]
+  source = "data"
+  destination = "/data"
+  auto_extend_size_threshold = 80
+  auto_extend_size_increment = "1GB"
+  auto_extend_size_limit = "10GB"
+

test/results/nginx/docker-entrypoint

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 # If running the production procfile then create or migrate existing database
-if [ "${*}" == "foreman start --procfile=Procfile.prod" ]; then
+if [ "${@: -3:1}" == "foreman" ] && [ "${@: -2:1}" == "start" ] && [ "${@: -1:1}" == "--procfile=Procfile.prod" ]; then
   ./bin/rails db:prepare
 fi
 

test/results/precompile_defer/docker-entrypoint

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 # If running the rails server then create or migrate existing database
-if [ "${*}" == "./bin/thrust ./bin/rails server" ]; then
+if [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ]; then
   ./bin/rails assets:precompile
   ./bin/rails db:prepare
 fi

test/results/swap/docker-entrypoint

@@ -13,7 +13,7 @@ if [ $UID -eq 0 ]; then
 fi
 
 # If running the rails server then create or migrate existing database
-if [ "${*}" == "./bin/thrust ./bin/rails server" ]; then
+if [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ]; then
   ./bin/rails db:prepare
 fi
 

test/test_litefs.rb

@@ -6,7 +6,7 @@ class TestLitefs < TestBase
   @rails_options = "--database=sqlite3"
   @generate_options = "--litefs"
 
-  def test_sqlite3
+  def test_litefs
     check_dockerfile
     check_entrypoint
     check_litefs

test/test_litestack.rb

@@ -10,7 +10,7 @@ def app_setup
     system "bundle add litestack"
   end
 
-  def test_sqlite3
+  def test_litestack
     check_dockerfile
   end
 end

test/test_litestream.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+class TestLitestream < TestBase
+  @rails_options = "--database=sqlite3"
+  @generate_options = "--litestream"
+
+  def app_setup
+    FileUtils.touch "fly.toml"
+  end
+
+  def test_litestream
+    check_dockerfile
+    check_entrypoint
+    check_toml
+  end
+end