Don't buffer stdout

Reviewed-on: #2

.gitea/workflows/build.yml

@@ -0,0 +1,27 @@
+name: build
+on:
+  pull_request:
+jobs:
+  build:
+    container:
+      image: node:24-bookworm
+      env:
+        USER: node
+        HOME: /home/node
+        NIX_CONFIG: experimental-features = nix-command flakes
+      volumes:
+        - /var/lib/pbri/docker/nix_runner_nix:/nix
+        - /var/lib/pbri/docker/nix_runner_etc:/etc/nix
+        - /var/lib/pbri/docker/nix_runner_home_node:/home/node
+      options: --user 1000:1000
+    steps:
+      - name: Prep nix env
+        run: |
+          if [ ! -f ~/.nix-profile/etc/profile.d/nix.sh ]; then
+            curl -L https://nixos.org/nix/install | bash -s -- --no-daemon
+          fi
+          . ~/.nix-profile/etc/profile.d/nix.sh 
+          env >> "$GITHUB_ENV"
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - run: nix build --log-format raw --print-build-logs .#yore

.gitea/workflows/deliver.yml

@@ -0,0 +1,40 @@
+name: deliver
+on:
+  push:
+    tags:
+      - 'v*'
+jobs:
+  deliver:
+    container:
+      image: node:24-bookworm
+      env:
+        USER: node
+        HOME: /home/node
+        NIX_CONFIG: experimental-features = nix-command flakes
+      volumes:
+        - /var/lib/pbri/docker/nix_runner_nix:/nix
+        - /var/lib/pbri/docker/nix_runner_etc:/etc/nix
+        - /var/lib/pbri/docker/nix_runner_home_node:/home/node
+      options: --user 1000:1000
+    steps:
+      - name: Prep nix env
+        run: |
+          if [ ! -f ~/.nix-profile/etc/profile.d/nix.sh ]; then
+            curl -L https://nixos.org/nix/install | bash -s -- --no-daemon
+          fi
+          . ~/.nix-profile/etc/profile.d/nix.sh
+          env >> "$GITHUB_ENV"
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Get image meta
+        run: nix build .#yore-meta -o /tmp/yore-meta
+      - name: Version check
+        run: |
+          VERSION=$(cat /tmp/yore-meta/version)
+          [ "$GITHUB_REF_NAME" = v"$VERSION" ]
+      - run: nix build --log-format raw --print-build-logs .#image -o image.tar.gz
+      - run: gunzip -c image.tar.gz > image.tar
+      - run: nix run .#crane -- auth login git.pbrinkmeier.de -u paul -p "$PASSWORD"
+        env:
+          PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
+      - run: nix run .#crane -- push image.tar $(cat /tmp/yore-meta/image-tag)

README.md

@@ -25,7 +25,7 @@ You can also use it to run a database for development by choosing a `$COMMAND` t
 ```
 BASE_DIR=./pgdata scripts/with-db psql "dbname=yore-test port=5433 user=yore-test"
 # cabal repl with env vars set
-BASE_DIR=./pgdata scripts/with-db scripts/dev
+BASE_DIR=./pgdata scripts/with-db cabal run
 ```
 
 By setting `$BASE_DIR` you can persist the database for later runs.

app/Main.hs

@@ -17,14 +17,12 @@ import Control.Monad.Trans.Except (ExceptT (..), catchE, runExceptT)
 import Data.Bifunctor (Bifunctor (..))
 import Data.Proxy (Proxy (..))
 import Data.Text (Text)
-import Data.Time (Day, LocalTime (..), TimeOfDay (..), ZonedTime (..), addDays, toGregorian)
+import Data.Time (Day, LocalTime (..), TimeOfDay (..), ZonedTime (..), addDays, nominalDiffTimeToSeconds, toGregorian)
 import GHC.Generics (Generic)
-import Network.HTTP.Media ((//), (/:))
 import Network.HTTP.Types (Status (..))
-import Network.Wai (Request (..))
+import Network.Wai (Middleware, Request (..), responseStatus)
 import Servant
-  ( Accept (..)
-  , Capture
+  ( Capture
   , Get
   , Handler
   , MimeRender (..)
@@ -59,12 +57,15 @@ import Yore.Download (downloadInto)
 import Yore.Error (Error (..))
 import Yore.Schedule (schedule)
 import Yore.Scrape (Issue (..), getIssuesByDay)
-import Yore.Servant (GetSendfile, Sendfile (..))
+import Yore.Servant (GetSendfile, HTML, Sendfile (..))
 import Yore.Time (addYears, getCurrentDay, getSecondsUntilMidnight)
 
+import Data.Fixed (Pico)
+import Data.Time.Clock.POSIX (getPOSIXTime)
 import qualified Envy
 import qualified Yore.DB as DB
 import qualified Yore.Log as Log
+import System.IO (hSetBuffering, BufferMode (..), stdout)
 
 data ConfigT f = Config
   { yorePort :: f =@@ Int ? 3000
@@ -79,10 +80,12 @@ deriving instance Show Config
 
 main :: IO ()
 main = do
+  hSetBuffering stdout NoBuffering
+
   cfg <-
     Envy.load @ConfigT >>= \case
-      Left err -> do
-        Log.error $ printf "failed to read config: %s" err
+      Left errs -> do
+        forM_ errs $ Log.error . printf "failed to read config: %s"
         exitFailure
       Right c ->
         pure c
@@ -114,24 +117,29 @@ doIndex cfg db =
 
 runServer :: Config -> DB -> IO ()
 runServer cfg db =
-  Warp.runSettings settings $ serve (Proxy @API) $ hoistServer (Proxy @API) nt $ server cfg db
+  Warp.runSettings settings $ loggerMiddleware $ serve (Proxy @API) $ hoistServer (Proxy @API) nt $ server cfg db
   where
     settings =
       foldr
         ($)
         Warp.defaultSettings
-        [ Warp.setLogger logger
-        , Warp.setPort cfg.yorePort
+        [ Warp.setPort cfg.yorePort
         , Warp.setBeforeMainLoop $ Log.info $ printf "listening on port %d" cfg.yorePort
         , Warp.setOnException onException
         ]
 
-    logger req status _ = do
-      Log.info $
-        printf
-          "%d %s"
-          (statusCode status)
-          (requestLine req)
+    logger :: Request -> Status -> Pico -> IO ()
+    logger req status s = do
+      Log.info $ printf "%d %s %ss" (statusCode status) (requestLine req) (formatMetric s)
+
+    loggerMiddleware :: Middleware
+    loggerMiddleware app req respond = do
+      begin <- getPOSIXTime
+      app req $ \res -> do
+        rr <- respond res
+        end <- getPOSIXTime
+        logger req (responseStatus res) (nominalDiffTimeToSeconds (end - begin))
+        pure rr
 
     onException mbReq ex = do
       Log.error $ printf "unhandled exception%s: %s%s" (maybe "" ((" in " ++) . requestLine) mbReq) (show ex)
@@ -143,6 +151,18 @@ runServer cfg db =
         (BS8.unpack $ requestMethod req)
         (BS8.unpack $ rawPathInfo req <> rawQueryString req)
 
+formatMetric :: Pico -> String
+formatMetric x = go (-9) prefixes
+  where
+    prefixes = ["p", "n", "u", "m", "", "k", "M", "G"]
+
+    go :: Int -> [String] -> String
+    go _ [] = show x
+    go e (p : ps)
+      | x < 10 ^^ e =
+          show @Int (floor $ x * 10 ^^ (-e + 3)) ++ p
+      | otherwise = go (e + 3) ps
+
 nt :: ExceptT Error IO a -> Handler a
 nt action = MkHandler $ do
   res <- runExceptT action
@@ -178,13 +198,13 @@ server cfg db = rootR :<|> todayR :<|> apiTodayR
     rootR = todayR 0
 
     todayR issue = do
-      dateThen <- ExceptT get100YearsAgo
+      dateThen <- getTodayWithOffset (-100) 0
       count <- DB.withConn db $ DB.getNumberOfIssues dateThen
       dayFile <- DB.withConn db $ DB.getDayFileByIssue dateThen issue
       pure $ RootModel dateThen dayFile issue count
 
     apiTodayR issue = do
-      dateThen <- ExceptT get100YearsAgo
+      dateThen <- getTodayWithOffset (-100) 0
       dayFile <- DB.withConn db $ DB.getDayFileByIssue dateThen issue
       let fullPath = cfg.yoreDownloadDir </> dayFile.relative_path
       secondsUntilMidnight <- liftIO getSecondsUntilMidnight
@@ -232,14 +252,6 @@ instance MimeRender HTML RootModel where
 
 -- Utils
 
-data HTML
-
-instance Accept HTML where
-  contentType _ = "text" // "html" /: ("charset", "utf-8")
-
-get100YearsAgo :: IO (Either Error Day)
-get100YearsAgo = runExceptT $ getTodayWithOffset (-100) 0
-
 getTodayWithOffset :: Integer -> Integer -> ExceptT Error IO Day
 getTodayWithOffset yearOffset dayOffset =
   ExceptT $

db/schema.sql

@@ -1,4 +1,4 @@
-\restrict VcBcXJr0fe2fRf2im7J0PCentZxDeYMChdNcReN5LtHeQrg2nXyh5MelIBAiGsO
+\restrict CAC30PXzQJosjH02Spia1vzj9Ui3LB1Cq2HI8r0A4eu33HP8ngxbbM2SzRfTj0c
 
 -- Dumped from database version 17.6
 -- Dumped by pg_dump version 17.6
@@ -129,7 +129,7 @@ ALTER TABLE ONLY yore.day_file
 -- PostgreSQL database dump complete
 --
 
-\unrestrict VcBcXJr0fe2fRf2im7J0PCentZxDeYMChdNcReN5LtHeQrg2nXyh5MelIBAiGsO
+\unrestrict CAC30PXzQJosjH02Spia1vzj9Ui3LB1Cq2HI8r0A4eu33HP8ngxbbM2SzRfTj0c
 
 
 --

flake.lock

@@ -64,11 +64,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1756064767,
-        "narHash": "sha256-SaJK9VYc/oF5aBSMSW+We8TggER4+uTAHvc2ztnLpYo=",
+        "lastModified": 1759790870,
+        "narHash": "sha256-rEsaO38v2wvAThPzgbYPgnv29mu6GxTYLjgci3GYaO4=",
         "ref": "main",
-        "rev": "af16429d827816cf9bec69026e13f50acc3d2691",
-        "revCount": 65,
+        "rev": "4db509c1b52ee7b5012abfef5e11e62beab27a16",
+        "revCount": 66,
         "type": "git",
         "url": "https://git.pbrinkmeier.de/paul/opium"
       },

flake.nix

@@ -26,11 +26,18 @@
         };
 
         # For building the package, we use only the files not ignored by Git as inputs.
-        # Also, flake.nix and flake.lock are not included to avoid rebuilds when
+        # Also, flake.nix, flake.lock and .gitea are not included to avoid rebuilds when
         # working on them.
         src = pkgs.lib.cleanSourceWith {
-          src = gitignore.lib.gitignoreSource ./.;
-          filter = path: type: builtins.elem (builtins.baseNameOf path) [ "flake.nix" "flake.lock" ];
+          src = ./.;
+          filter = gitignore.lib.gitignoreFilterWith {
+            basePath = ./.;
+            extraRules = ''
+              flake.nix
+              flake.lock
+              .gitea
+            '';
+          };
         };
 
         yore = pkgs.lib.pipe
@@ -51,16 +58,37 @@
             # Return only the bin folder to curb image size
             compose.justStaticExecutables
           ];
+
+        image =
+          pkgs.dockerTools.buildImage {
+            name = "git.pbrinkmeier.de/paul/yore";
+            tag = yore.version;
+            copyToRoot = [ pkgs.cacert ];
+            config.Cmd = [ "${entrypoint}/bin/run" ];
+          };
+
+        entrypoint = pkgs.writeShellApplication {
+          name = "run";
+          runtimeInputs = [ pkgs.coreutils pkgs.dbmate yore ];
+          text = ''
+            mkdir -p /workspace/db
+            cp -r ${./db}/* /workspace/db
+            cd /workspace
+            dbmate up
+
+	    yore
+          '';
+        };
       in {
         packages = rec {
-          docker =
-            pkgs.dockerTools.buildImage {
-              name = "git.pbrinkmeier.de/paul/yore";
-              tag = yore.version;
-              config.Cmd = [ "${yore}/bin/yore" ];
-            };
-
           default = yore;
+          inherit yore image;
+          yore-meta = pkgs.runCommand "yore-meta" {} ''
+            mkdir -p $out
+            echo -n ${yore.version} > $out/version
+            echo -n ${image.drvAttrs.imageName}:${image.drvAttrs.imageTag} > $out/image-tag
+          '';
+
 
           opium_ = opium.packages.${system}.opium;
         };
@@ -82,5 +110,11 @@
               overrides = addOpium;
             }).env;
         };
+        apps = {
+          crane = {
+            type = "app";
+            program = "${pkgs.crane}/bin/crane";
+          };
+        };
       });
 }

src/Envy.hs

@@ -56,13 +56,13 @@ type t ? d = Optional t d
 load
   :: forall (m :: ConfigVariant -> Type)
    . (Generic (m Value), GFromEnv (Rep (m Spec)) (Rep (m Value)))
-  => IO (Either String (m Value))
+  => IO (Either [String] (m Value))
 load = do
   env <- getEnvironment
   pure $ to <$> gFromEnv @(Rep (m Spec)) @(Rep (m Value)) Proxy env
 
 class GFromEnv spec value where
-  gFromEnv :: Proxy (spec c) -> [(String, String)] -> Either String (value c)
+  gFromEnv :: Proxy (spec c) -> [(String, String)] -> Either [String] (value c)
 
 instance
   (GFromEnv i o)
@@ -77,7 +77,15 @@ instance
   gFromEnv Proxy env = M1 <$> gFromEnv @i @o Proxy env
 
 instance (GFromEnv i1 o1, GFromEnv i2 o2) => GFromEnv (i1 :*: i2) (o1 :*: o2) where
-  gFromEnv Proxy env = (:*:) <$> gFromEnv @i1 @o1 Proxy env <*> gFromEnv @i2 @o2 Proxy env
+  gFromEnv Proxy env =
+    case (eLhs, eRhs) of
+      (Left errs1, Right _) -> Left errs1
+      (Right _, Left errs2) -> Left errs2
+      (Left errs1, Left errs2) -> Left $ errs1 ++ errs2
+      (Right lhs, Right rhs) -> Right $ lhs :*: rhs
+    where
+      eLhs = gFromEnv @i1 @o1 Proxy env
+      eRhs = gFromEnv @i2 @o2 Proxy env
 
 instance
   (KnownSymbol sym, EnvVarSpec s t)
@@ -86,7 +94,7 @@ instance
       (M1 S meta2 (Rec0 t))
   where
   gFromEnv Proxy env =
-    M1 . K1 <$> decodeEnvVar @s @t Proxy varName (lookup varName env)
+    M1 . K1 <$> first (: []) (decodeEnvVar @s @t Proxy varName (lookup varName env))
     where
       varName = selectorNameToEnvVarName $ symbolVal $ Proxy @sym
 

src/Yore/Servant.hs

@@ -1,11 +1,13 @@
 {-# LANGUAGE OverloadedRecordDot #-}
+{-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE TypeFamilies #-}
 
-module Yore.Servant (GetSendfile, Sendfile (..)) where
+module Yore.Servant (GetSendfile, Sendfile (..), HTML) where
 
+import Network.HTTP.Media ((//), (/:))
 import Network.HTTP.Types (ResponseHeaders, status200)
 import Network.Wai (responseFile)
-import Servant (HasServer (..))
+import Servant (Accept (..), HasServer (..))
 import Servant.Server.Internal.Delayed (runAction)
 import Servant.Server.Internal.RouteResult (RouteResult (..))
 import Servant.Server.Internal.Router (Router' (..))
@@ -29,3 +31,8 @@ instance HasServer GetSendfile context where
   route _ _ action = RawRouter $ \env request respond ->
     runAction action env request respond $ \sendfile ->
       Route $ responseFile status200 sendfile.headers sendfile.path Nothing
+
+data HTML
+
+instance Accept HTML where
+  contentType _ = "text" // "html" /: ("charset", "utf-8")

yore.cabal

@@ -1,7 +1,7 @@
 cabal-version: 3.4
 
 name:           yore
-version:        0
+version:        0.0.3
 author:         Paul Brinkmeier
 maintainer:     hallo@pbrinkmeier.de
 copyright:      2023 Paul Brinkmeier
@@ -50,6 +50,7 @@ library
     , directory
     , filepath
     , html-parse
+    , http-media
     , http-types
     , lens
     , modern-uri
@@ -81,7 +82,6 @@ executable yore
     , bytestring
     , filepath
     , opium
-    , http-media
     , http-types
     , servant-server
     , text