For web server, aleph which is build on top of netty is superb as it is async in nature and has cool instrumentation capabilities if one requires such kind of optimisation, which by default wouldn't be much of a concern. We will use compojure for routing. Any typical web service having a handful of routes would be fast enough with compojure which serves its purpose well. As for logging using logging context, we will use logback because it works well when building an uberjar as opposed to log4j2.
As for configuration, we will keep it in properties file. Both the logging and app config will use the same properties file. If we require multiple properties file as in base properties and environment specific ones, we can run a pre-init script to combine them and pass it to the service to bootstrap. Now to the code part.
1. Create a
lein project. The folder structure is similar to below.bootstrap-clj2. We will use
.
├── README.md
├── config
│ ├── dev.properties
│ └── prod.properties
├── logs
├── project.clj
├── resources
│ ├── base.properties
│ └── logback.xml
├── scripts
│ ├── boot.properties
│ └── fusion.clj
├── src
│ └── com
│ └── qlambda
│ └── bootstrap
│ ├── constants.clj
│ ├── logger.clj
│ ├── services
│ │ └── persistance.clj
│ ├── utils.clj
│ └── web.clj
└── test
lein-shell to execute the fusion.clj which will combine the config files. This can be written in any language and called into from lein.;; project.clj3. This is the boot-clj script which combines the config. The combined config will be present in
(defproject bootstap "0.0.1"
;; ...
:main com.qlambda.bootstrap.web
:prep-tasks [["shell" "boot" "scripts/fusion.clj"] "javac" "compile"]
:profiles {:dev {:repl-options {:init-ns com.qlambda.bootstrap.web}}
:uberjar {:aot :all
:main ^skip-aot com.qlambda.bootstrap.web
:jvm-opts ["-server" "-XX:+UnlockCommercialFeatures" "-XX:+FlightRecorder"]}}
:source-paths ["src"]
:test-paths ["test"]
:target-path "target/%s"
:jvm-opts ["-server"]
:min-lein-version "2.7.1"
:plugins [[lein-shell "0.5.0"]]
;; ..)
config/{env}/config.properties file.#!/usr/bin/env boot4. The web layer which does the service initialisation.
;; Fuses base config with APP_ENV exported config.
;; Part of build pipeline.
(set-env!
:source-paths #{"src"}
:dependencies '[[org.clojure/clojure "1.9.0"]
[ch.qos.logback/logback-core "1.2.3"]
[ch.qos.logback/logback-classic "1.2.3"]
[org.clojure/tools.logging "0.4.0"]])
(require '[clojure.tools.logging :as log]
'[clojure.java.io :as io])
(import '[java.util Properties]
'[java.io File])
(defn get-abs-path []
(.getAbsolutePath (File. "")))
(def base-config-path "resources/base.properties")
(def config-dir (format "%s/config" (get-abs-path)))
(def env-xs ["dev" "prod"]) ; envs for which config will be generated
(defn prop-comment [env]
(format "App config (fusion generated) for %s.\nSome things are not ours to tamper with." env))
(defn unified-config [env]
(format "%s/%s/config.properties" config-dir env))
(defn store-config-properties [unified-props env]
(log/info (str "Unified config path: " (unified-config env)))
(with-open [writer (io/writer (unified-config env))]
(.store unified-props writer (prop-comment env)))
(log/info (str "Bosons generated for " env)))
(defn create-env-config-dirs [env-xs]
(doall (map (fn [env]
(let [env-dir (format "%s/%s" config-dir env)]
(when-not (.exists (File. env-dir))
(.mkdirs (File. env-dir))))) env-xs)))
(defn load-config-properties []
(log/info "Loading particles.")
(create-env-config-dirs env-xs)
(let [prop-obj (doto (Properties.) (.load (io/input-stream base-config-path)))
app-props (map (fn [env] (doto (Properties.) (.load (io/input-stream (str config-dir "/" env ".properties"))))) env-xs)
unified-props (map (fn [env-props]
(doto (Properties.)
(.putAll prop-obj)
(.putAll env-props))) app-props)]
(doall (map #(store-config-properties %1 %2) unified-props env-xs))
(log/info "Fusion process complete.")))
(defn -main [& args]
(log/info "Starting particle fusion system.")
(load-config-properties))
;; web.clj
(ns com.qlambda.bootstrap.web
"Web Layer"
(:require [aleph.http :as http]
[compojure.core :as compojure :refer [GET POST defroutes]]
[ring.middleware.params :refer [wrap-params]]
[ring.middleware.reload :refer [wrap-reload]]
[ring.middleware.keyword-params :refer [wrap-keyword-params]]
[ring.middleware.multipart-params :refer [wrap-multipart-params]]
[clojure.java.io :as io]
[clojure.tools.nrepl.server :as nrepl]
[clojure.tools.namespace.repl :as repl]
[com.qlambda.bootstrap.constants :as const]
[com.qlambda.bootstrap.utils :refer [parse-int] :as utils]
[com.qlambda.bootstrap.logger :as log])
(:import [java.util Properties])
(:gen-class))
(defn test-handler
[req]
(log/info "test-handler")
{:status 200
:body "ok"})
(defroutes app-routes
(POST ["/test"] {} test-handler))
(def app
(-> app-routes
(utils/wrap-logging-context)
(wrap-keyword-params)
(wrap-params)
(wrap-multipart-params)
(wrap-reload)))
; --- REPL ---
(defn start-nrepl-server []
(deliver const/nrepl-server (nrepl/start-server :port (parse-int (:nrepl-port @const/config)))))
; --- Config ---
(defn load-config []
(let [cfg-path (first (filter #(not-empty %1) [(System/getenv "APP_CONFIG") const/default-config-path]))]
(doto (Properties.)
(.load (io/input-stream cfg-path)))))
(defn -main
"Start the service."
[& args]
(deliver const/config (utils/keywordize-properties (load-config)))
(start-nrepl-server)
(log/info "Starting service.")
;; other initializations ..
(http/start-server #'app {:port (parse-int (:server-port @const/config))})
(log/info "Service started."))
;; constants.clj
(ns com.qlambda.bootstrap.constants
"App constants and configs"
(:require [clojure.tools.namespace.repl :as repl]))
(repl/disable-reload!)
(def config (promise)) ;keywordized properties map
(def nrepl-server (promise))
(def default-config-path "config/config.properties")
;; utils.clj5. The logger module helps to log arbitrary objects as json using logback markers as well. The custom logger is used in this example with
(ns com.qlambda.bootstrap.utils
"Helper functions"
(:require [aleph.http :as http]
[wharf.core :as char-trans]
[com.qlambda.bootstrap.constants :as const]
[com.qlambda.bootstrap.logger :as log]))
(defn parse-int [str-num]
(try
(Integer/parseInt str-num)
(catch Exception ex 0)))
(defn keywordize-properties
"Converts a properties map to clojure hashmap with keyword keys"
[props]
(into {} (for [[k v] props] [(keyword (str/replace k #"\." "-")) v])))
(defn wrap-logging-context [handler]
(fn [request]
(binding [log/ctx-headers (char-trans/transform-keys char-trans/hyphen->underscore (:headers request))]
(handler request))))
net.logstash.logback.encoder.LogstashEncoder so that the logs can be pumped to Elasticsearch via logstash keeping the custom ELK data format.(ns com.qlambda.bootstrap.loggerAn example log generated is of the format
"Logging module"
(:import [net.logstash.logback.marker Markers]
[org.slf4j Logger LoggerFactory]))
(declare ^:dynamic ctx-headers)
(defn marker-append
"Marker which is used to log arbitrary objects with the given string as key to JSON"
[ctx-coll marker]
(if-let [lst (not-empty (first ctx-coll))]
(recur (rest ctx-coll) (.with marker (Markers/append (first lst) (second lst))))
marker))
(defmacro log-with-marker [level msg ctx]
`(let [logger# (LoggerFactory/getLogger ~(str *ns*))
header-mrkr# (Markers/append "headers_data" ctx-headers)]
(condp = ~level
:trace (when (.isTraceEnabled logger#) (.trace logger# (marker-append ~ctx header-mrkr#) ~msg))
:debug (when (.isDebugEnabled logger#) (.debug logger# (marker-append ~ctx header-mrkr#) ~msg))
:info (when (.isInfoEnabled logger#) (.info logger# (marker-append ~ctx header-mrkr#) ~msg))
:warn (when (.isWarnEnabled logger#) (.warn logger# (marker-append ~ctx header-mrkr#) ~msg))
:error (when (.isErrorEnabled logger#) (.error logger# (marker-append ~ctx header-mrkr#) ~msg)))))
(defmacro trace
([msg]
`(log-with-marker :trace ~msg []))
([msg ctx]
`(log-with-marker :trace ~msg [~ctx])))
(defmacro debug
([msg]
`(log-with-marker :debug ~msg []))
([msg ctx]
`(log-with-marker :debug ~msg [~ctx])))
(defmacro info
([msg]
`(log-with-marker :info ~msg []))
([msg ctx]
`(log-with-marker :info ~msg [~ctx])))
(defmacro warn
([msg]
`(log-with-marker :warn ~msg []))
([msg ctx]
`(log-with-marker :warn ~msg [~ctx])))
(defmacro error
([msg]
`(log-with-marker :error ~msg []))
([msg ctx]
`(log-with-marker :error ~msg [~ctx])))
{"@timestamp":"2018-07-24T18:44:47.876+00:00","description":"test-handler","logger":"com.qlambda.bootstrap.web","thread":"manifold-pool-2-3","level":"INFO","level":20000,"headers_data":{"host":"localhost:8080","user_agent":"PostmanRuntime/6.4.1","content_type":"application/x-www-form-urlencoded","content_length":"48","connection":"keep-alive","accept":"*/*","accept_encoding":"gzip, deflate","cache_control":"no-cache"},"data_version":2,"type":"log","roletype":"bootstrap","category":"example","service":"bootstrap","application_version":"0.0.1","application":"bootstrap","environment":"dev"}
6. The logback config xml file follows. This uses the config from properties file specified by the APP_CONFIG.<?xml version="1.0" encoding="UTF-8"?>7. Sample config files, which will be combined and the properties in the later ones replaces the parent properties.
<configuration debug="${log.config.debug:-false}">
<property file="${APP_CONFIG}" />
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE">
<File>${log.file}</File>
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<customFields>{"data_version":2,"type":"log","roletype":"${app.roletype}","category":"${app.category}","service":"${app.service}","application_version":"${app.version}","application":"${app.name}","environment":"${env}"}</customFields>
<fieldNames>
<timestamp>@timestamp</timestamp>
<levelValue>level</levelValue>
<thread>thread</thread>
<message>description</message>
<logger>logger</logger>
<version>[ignore]</version>
</fieldNames>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<maxIndex>10</maxIndex>
<FileNamePattern>logs/foo.json.log.%i</FileNamePattern>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<MaxFileSize>${log.max.filesize:-1GB}</MaxFileSize>
</triggeringPolicy>
</appender>
<appender class="ch.qos.logback.core.ConsoleAppender" name="CONSOLE">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{HH:mm:ss.SSS} %green([%t]) %highlight(%level) %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger additivity="false" level="${log.bootstrap.level:-debug}" name="com.qlambda.bootstrap">
<appender-ref ref="FILE"/>
<appender-ref ref="CONSOLE"/>
</logger>
<root level="${log.root.level:-info}">
<appender-ref ref="FILE"/>
<appender-ref ref="CONSOLE"/>
</root>
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
</configuration>
# base.properties
# --- app props ---
app.roletype=bootstrap
app.category=example
app.service=bootstrap
# ...
# --- server props ---
server.port=8080
nrepl.port=8081
# --- logging ---
log.bootstrap.level=debug
log.root.level=info
log.config.debug=false
log.max.filesize=1GB
log.max.history.days=3
log.archive.totalsize=10GB
log.prune.on.start=false
log.immediate.flush=true
log.file=logs/app.json.log
# dev.properties8. Couple of helper shell scripts
# --- app props ---
env=dev
#!/bin/bash
# run
function startServer {
echo -e "\nStarting server .."
lein do clean
export APP_ENV="dev"
export APP_CONFIG="config/dev/config.properties"
lein run
}
startServer
#!/bin/bash
# nrepl
lein repl :connect localhost:8081
#!/bin/bash
# repl
export APP_ENV="dev"
export APP_CONFIG="config/dev/config.properties" # const: will be auto-generated
lein repl
#!/bin/bashThe above code snippets would help in bootstapping a production grade clojure web service in no time with a great deal of flexibility. As for instrumentation, new relic does a great job. And this does not require any Mary Freaking Fancy Poppins frameworks which is more trouble than being helpful.
# release
function release {
echo -e "\nGenerating uberjar .."
lein do clean
export APP_ENV="dev"
export APP_CONFIG="config/dev/config.properties"
lein uberjar
}
release
