From b04d09e0dd5878741d67f7a5927f71a08a9099e0 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Sun, 31 Dec 2017 17:23:33 +0000 Subject: [PATCH 01/38] * Builds with `process` 1.6 * add dependency for `tls-session-manager` * bump resolver --- ChangeLog.md | 5 +++++ Data/Conduit/Process/Unix.hs | 4 +++- Keter/Proxy.hs | 12 +++++++----- Keter/Types/V04.hs | 4 ++++ Keter/Types/V10.hs | 7 +++++-- etc/keter-config.yaml | 1 + keter.cabal | 7 ++++--- stack.yaml | 2 +- 8 files changed, 30 insertions(+), 12 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index a45e7ce..3f170a5 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,8 @@ +## 1.5 + +* Builds with `process` 1.6 +* add dependency for `tls-session-manager` + ## 1.4.3.1 * Add cabal flag `system-filepath` for compatibility with older versions of fsnotify. diff --git a/Data/Conduit/Process/Unix.hs b/Data/Conduit/Process/Unix.hs index 8f62625..4ec53a2 100644 --- a/Data/Conduit/Process/Unix.hs +++ b/Data/Conduit/Process/Unix.hs @@ -61,7 +61,9 @@ import System.Process.Internals (ProcessHandle (..), ProcessHandle__ (..)) processHandleMVar :: ProcessHandle -> MVar ProcessHandle__ -#if MIN_VERSION_process(1, 2, 0) +#if MIN_VERSION_process(1, 6, 0) +processHandleMVar (ProcessHandle m _ _) = m +#elif MIN_VERSION_process(1, 2, 0) processHandleMVar (ProcessHandle m _) = m #else processHandleMVar (ProcessHandle m) = m diff --git a/Keter/Proxy.hs b/Keter/Proxy.hs index a903000..b3d0c47 100644 --- a/Keter/Proxy.hs +++ b/Keter/Proxy.hs @@ -49,6 +49,7 @@ import Network.Wai.Application.Static (defaultFileServerSettings, ssListing, staticApp) import qualified Network.Wai.Handler.Warp as Warp import qualified Network.Wai.Handler.WarpTLS as WarpTLS +import qualified Network.TLS.SessionManager as TLSSession import Network.Wai.Middleware.Gzip (gzip, GzipSettings(..), GzipFiles(..)) import Prelude hiding (FilePath, (++)) import WaiAppStatic.Listing (defaultListing) @@ -72,15 +73,15 @@ reverseProxy useHeader timeBound manager hostLookup listener = (run, isSecure) = case listener of LPInsecure host port -> (Warp.runSettings (warp host port), False) - LPSecure host port cert chainCerts key -> (WarpTLS.runTLS - (connectClientCertificates hostLookup $ WarpTLS.tlsSettingsChain + LPSecure host port cert chainCerts key session -> (WarpTLS.runTLS + (connectClientCertificates hostLookup session $ WarpTLS.tlsSettingsChain cert (V.toList chainCerts) key) (warp host port), True) -connectClientCertificates :: HostLookup -> WarpTLS.TLSSettings -> WarpTLS.TLSSettings -connectClientCertificates hl s = +connectClientCertificates :: HostLookup -> Bool -> WarpTLS.TLSSettings -> WarpTLS.TLSSettings +connectClientCertificates hl session s = let newHooks@TLS.ServerHooks{..} = WarpTLS.tlsServerHooks s -- todo: add nested lookup @@ -89,7 +90,8 @@ connectClientCertificates hl s = newOnServerNameIndication Nothing = return mempty -- we could return default certificate here in - s { WarpTLS.tlsServerHooks = newHooks{TLS.onServerNameIndication = newOnServerNameIndication}} + s { WarpTLS.tlsServerHooks = newHooks{TLS.onServerNameIndication = newOnServerNameIndication} + , WarpTLS.tlsSessionManagerConfig = if session then (Just TLSSession.defaultConfig) else Nothing } withClient :: Bool -- ^ is secure? -> Bool -- ^ use incoming request header for IP address diff --git a/Keter/Types/V04.hs b/Keter/Types/V04.hs index 444419a..e4fda5b 100644 --- a/Keter/Types/V04.hs +++ b/Keter/Types/V04.hs @@ -5,6 +5,7 @@ module Keter.Types.V04 where import Control.Applicative import Data.Aeson +import Data.Bool import Data.Conduit.Network (HostPreference) import Data.Default import qualified Data.Set as Set @@ -15,6 +16,7 @@ import Keter.Types.Common import Network.HTTP.ReverseProxy.Rewrite import qualified Network.Wai.Handler.Warp as Warp import qualified Network.Wai.Handler.WarpTLS as WarpTLS +import qualified Network.TLS.SessionManager as TLSSession import Prelude hiding (FilePath) data AppConfig = AppConfig @@ -121,6 +123,7 @@ instance ParseYamlFile TLSConfig where key <- lookupBase basedir o "key" host <- (fmap fromString <$> o .:? "host") .!= "*" port <- o .:? "port" .!= 443 + session <- bool Nothing (Just TLSSession.defaultConfig) <$> o .:? "session" .!= False return $! TLSConfig ( Warp.setHost host $ Warp.setPort port @@ -128,6 +131,7 @@ instance ParseYamlFile TLSConfig where WarpTLS.defaultTlsSettings { WarpTLS.certFile = cert , WarpTLS.keyFile = key + , WarpTLS.tlsSessionManagerConfig = session } -- | Controls execution of the nginx thread. Follows the settings type pattern. diff --git a/Keter/Types/V10.hs b/Keter/Types/V10.hs index 3e61efa..4aae232 100644 --- a/Keter/Types/V10.hs +++ b/Keter/Types/V10.hs @@ -17,7 +17,7 @@ import Data.Conduit.Network (HostPreference) import Data.Default import qualified Data.HashMap.Strict as HashMap import qualified Data.Map as Map -import Data.Maybe (catMaybes, fromMaybe) +import Data.Maybe (catMaybes, fromMaybe, isJust) import qualified Data.Set as Set import Data.String (fromString) import Data.Vector (Vector) @@ -70,6 +70,7 @@ instance ToJSON BundleConfig where data ListeningPort = LPSecure !HostPreference !Port !F.FilePath !(V.Vector F.FilePath) !F.FilePath + !Bool | LPInsecure !HostPreference !Port instance ParseYamlFile ListeningPort where @@ -77,6 +78,7 @@ instance ParseYamlFile ListeningPort where host <- (fmap fromString <$> o .:? "host") .!= "*" mcert <- lookupBaseMaybe basedir o "certificate" mkey <- lookupBaseMaybe basedir o "key" + session <- o .:? "session" .!= False case (mcert, mkey) of (Nothing, Nothing) -> do port <- o .:? "port" .!= 80 @@ -85,7 +87,7 @@ instance ParseYamlFile ListeningPort where port <- o .:? "port" .!= 443 chainCerts <- o .:? "chain-certificates" >>= maybe (return V.empty) (parseYamlFile basedir) - return $ LPSecure host port cert chainCerts key + return $ LPSecure host port cert chainCerts key session _ -> fail "Must provide both certificate and key files" data KeterConfig = KeterConfig @@ -127,6 +129,7 @@ instance ToCurrent KeterConfig where (WarpTLS.certFile ts) V.empty (WarpTLS.keyFile ts) + (isJust $ WarpTLS.tlsSessionManagerConfig ts) instance Default KeterConfig where def = KeterConfig diff --git a/etc/keter-config.yaml b/etc/keter-config.yaml index ed79e09..19bd6a1 100644 --- a/etc/keter-config.yaml +++ b/etc/keter-config.yaml @@ -15,6 +15,7 @@ listeners: #port: 443 key: key.pem certificate: certificate.pem + session: true # User to run applications as diff --git a/keter.cabal b/keter.cabal index 861c0f6..1cc6be0 100644 --- a/keter.cabal +++ b/keter.cabal @@ -1,5 +1,5 @@ Name: keter -Version: 1.4.3.2 +Version: 1.5 Synopsis: Web application deployment manager, focusing on Haskell web frameworks Description: Hackage documentation generation is not reliable. For up to date documentation, please see: . Homepage: http://www.yesodweb.com/ @@ -37,7 +37,7 @@ Library , template-haskell , blaze-builder >= 0.3 && < 0.5 , yaml >= 0.8.4 && < 0.9 - , unix-compat >= 0.3 && < 0.5 + , unix-compat >= 0.3 && < 0.6 , conduit >= 1.1 , conduit-extra >= 1.1 , http-reverse-proxy >= 0.4.2 && < 0.5 @@ -61,7 +61,8 @@ Library , stm >= 2.4 , async , lifted-base - , tls >= 1.3.4 + , tls >= 1.4 + , tls-session-manager if impl(ghc < 7.6) build-depends: ghc-prim diff --git a/stack.yaml b/stack.yaml index c6d4431..308d6ef 100644 --- a/stack.yaml +++ b/stack.yaml @@ -3,4 +3,4 @@ flags: packages: - '.' extra-deps: [] -resolver: lts-7.19 +resolver: lts-10.1 From b7a77c3acb26db3af4a10d971d0f8a505f37e48b Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Sat, 4 Mar 2017 12:56:43 +0000 Subject: [PATCH 02/38] Add per bundle certificates for static files and redirects. --- Keter/App.hs | 42 +++++++++++++++++++++--------------------- Keter/Proxy.hs | 1 + Keter/Types/V10.hs | 8 ++++++++ 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/Keter/App.hs b/Keter/App.hs index 44fb06e..8f1e066 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -2,6 +2,7 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TupleSections #-} module Keter.App ( App , AppStartConfig (..) @@ -132,26 +133,23 @@ withReservations asc aid bconfig f = withActions asc bconfig $ \wacs backs actio withActions :: AppStartConfig -> BundleConfig - -> ([WebAppConfig Port] -> [BackgroundConfig] -> Map Host (ProxyAction, TLS.Credentials) -> IO a) + -> ([ WebAppConfig Port] -> [BackgroundConfig] -> Map Host (ProxyAction, TLS.Credentials) -> IO a) -> IO a withActions asc bconfig f = loop (V.toList $ bconfigStanzas bconfig) [] [] Map.empty where + -- todo: add loading from relative location + loadCert (SSL certFile chainCertFiles keyFile) = + either (const mempty) (TLS.Credentials . (:[])) + <$> TLS.credentialLoadX509Chain certFile (V.toList chainCertFiles) keyFile + loadCert _ = return mempty + loop [] wacs backs actions = f wacs backs actions loop (Stanza (StanzaWebApp wac) rs:stanzas) wacs backs actions = bracketOnError - ( - getPort (ascLog asc) (ascPortPool asc) >>= either throwIO - (\p -> do - c <- case waconfigSsl wac of - -- todo: add loading from relative location - SSL certFile chainCertFiles keyFile -> - either (const mempty) (TLS.Credentials . (:[])) <$> - TLS.credentialLoadX509Chain certFile (V.toList chainCertFiles) keyFile - _ -> return mempty - return (p, c) - ) + (getPort (ascLog asc) (ascPortPool asc) >>= either throwIO + (\p -> fmap (p,) <$> loadCert $ waconfigSsl wac) ) - (\(port, cert) -> releasePort (ascPortPool asc) port) + (\(port, _) -> releasePort (ascPortPool asc) port) (\(port, cert) -> loop stanzas (wac { waconfigPort = port } : wacs) @@ -159,19 +157,21 @@ withActions asc bconfig f = (Map.unions $ actions : map (\host -> Map.singleton host ((PAPort port (waconfigTimeout wac), rs), cert)) hosts)) where hosts = Set.toList $ Set.insert (waconfigApprootHost wac) (waconfigHosts wac) - loop (Stanza (StanzaStaticFiles sfc) rs:stanzas) wacs backs actions0 = - loop stanzas wacs backs actions + loop (Stanza (StanzaStaticFiles sfc) rs:stanzas) wacs backs actions0 = do + cert <- loadCert $ sfconfigSsl sfc + loop stanzas wacs backs (actions cert) where - actions = Map.unions + actions cert = Map.unions $ actions0 - : map (\host -> Map.singleton host ((PAStatic sfc, rs), mempty)) + : map (\host -> Map.singleton host ((PAStatic sfc, rs), cert)) (Set.toList (sfconfigHosts sfc)) - loop (Stanza (StanzaRedirect red) rs:stanzas) wacs backs actions0 = - loop stanzas wacs backs actions + loop (Stanza (StanzaRedirect red) rs:stanzas) wacs backs actions0 = do + cert <- loadCert $ redirconfigSsl red + loop stanzas wacs backs (actions cert) where - actions = Map.unions + actions cert = Map.unions $ actions0 - : map (\host -> Map.singleton host ((PARedirect red, rs), mempty)) + : map (\host -> Map.singleton host ((PARedirect red, rs), cert)) (Set.toList (redirconfigHosts red)) loop (Stanza (StanzaReverseProxy rev mid to) rs:stanzas) wacs backs actions0 = loop stanzas wacs backs actions diff --git a/Keter/Proxy.hs b/Keter/Proxy.hs index b3d0c47..084c6be 100644 --- a/Keter/Proxy.hs +++ b/Keter/Proxy.hs @@ -163,6 +163,7 @@ withClient isSecure useHeader bound manager hostLookup = , redirconfigStatus = 301 , redirconfigActions = V.singleton $ RedirectAction SPAny $ RDPrefix True host' Nothing + , redirconfigSsl = SSLTrue } performAction req (PAPort port tbound) = diff --git a/Keter/Types/V10.hs b/Keter/Types/V10.hs index 4aae232..6ce3eab 100644 --- a/Keter/Types/V10.hs +++ b/Keter/Types/V10.hs @@ -238,6 +238,7 @@ data StaticFilesConfig = StaticFilesConfig -- FIXME basic auth , sfconfigMiddleware :: ![ MiddlewareConfig ] , sfconfigTimeout :: !(Maybe Int) + , sfconfigSsl :: !SSLConfig } deriving Show @@ -249,6 +250,7 @@ instance ToCurrent StaticFilesConfig where , sfconfigListings = True , sfconfigMiddleware = [] , sfconfigTimeout = Nothing + , sfconfigSsl = SSLFalse } instance ParseYamlFile StaticFilesConfig where @@ -258,6 +260,7 @@ instance ParseYamlFile StaticFilesConfig where <*> o .:? "directory-listing" .!= False <*> o .:? "middleware" .!= [] <*> o .:? "connection-time-bound" + <*> o .:? "ssl" .!= SSLFalse instance ToJSON StaticFilesConfig where toJSON StaticFilesConfig {..} = object @@ -266,12 +269,14 @@ instance ToJSON StaticFilesConfig where , "directory-listing" .= sfconfigListings , "middleware" .= sfconfigMiddleware , "connection-time-bound" .= sfconfigTimeout + , "ssl" .= sfconfigSsl ] data RedirectConfig = RedirectConfig { redirconfigHosts :: !(Set Host) , redirconfigStatus :: !Int , redirconfigActions :: !(Vector RedirectAction) + , redirconfigSsl :: !SSLConfig } deriving Show @@ -282,6 +287,7 @@ instance ToCurrent RedirectConfig where , redirconfigStatus = 301 , redirconfigActions = V.singleton $ RedirectAction SPAny $ RDPrefix False (CI.mk to) Nothing + , redirconfigSsl = SSLFalse } instance ParseYamlFile RedirectConfig where @@ -289,12 +295,14 @@ instance ParseYamlFile RedirectConfig where <$> (Set.map CI.mk <$> ((o .: "hosts" <|> (Set.singleton <$> (o .: "host"))))) <*> o .:? "status" .!= 303 <*> o .: "actions" + <*> o .:? "ssl" .!= SSLFalse instance ToJSON RedirectConfig where toJSON RedirectConfig {..} = object [ "hosts" .= Set.map CI.original redirconfigHosts , "status" .= redirconfigStatus , "actions" .= redirconfigActions + , "ssl" .= redirconfigSsl ] data RedirectAction = RedirectAction !SourcePath !RedirectDest From 99e9b47fdcba085a50d8adcc46d760c4c5d9476d Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Sat, 4 Mar 2017 20:11:25 +0000 Subject: [PATCH 03/38] update sample configuration --- incoming/foo1_0/config/keter.yaml | 39 ++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/incoming/foo1_0/config/keter.yaml b/incoming/foo1_0/config/keter.yaml index be1da10..e22d7ae 100644 --- a/incoming/foo1_0/config/keter.yaml +++ b/incoming/foo1_0/config/keter.yaml @@ -3,13 +3,13 @@ stanzas: exec: ../hello args: - Hello World v1.0 - #ssl : true -# ssl: -# key: /opt/keter/etc/cert/hello.key -# certificate: /opt/keter/etc/cert/hello.crt -# chain-certificates: -# - /opt/keter/etc/middle.crt -# - /opt/keter/etc/root.crt + # ssl : true + # ssl: + # key: /opt/keter/etc/cert/hello.key + # certificate: /opt/keter/etc/cert/hello.crt + # chain-certificates: + # - /opt/keter/etc/middle.crt + # - /opt/keter/etc/root.crt env: FROM_KETER_CONFIG: foo bar baz @@ -38,6 +38,16 @@ stanzas: requires-secure: true root: ../../ #connection-time-bound: 0 + # true will use the default certificate + # ssl: true + + # or we can have some additional certificate which supports unsafe1_1_0 + # ssl: + # key: /opt/keter/etc/cert/hello.key + # certificate: /opt/keter/etc/cert/hello.crt + # chain-certificates: + # - /opt/keter/etc/middle.crt + # - /opt/keter/etc/root.crt - type: static-files host: unsafe2_1_0 @@ -73,7 +83,20 @@ stanzas: # keter : rocks # - headers: # Access-Control-Allow-Origin : "*" - + - type: redirect + hosts: + - asome.com + - www.asome.com + - bsome.com + - www.bsome.com + actions: + - host: keter1_0 + secure: true + # ssl: + # key: /opt/certs/a_b_wa_wb/privkey.pem + # certificate: /opt/certs/a_b_wa_wb/cert.pem + # chain-certificates: + # - /opt/certs/a_b_wa_wb/chain.pem plugins: #postgres: true # Syntax for remote-DB server: From 2052ac7f57833c139d331a563a6af8b964554599 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Sat, 4 Mar 2017 21:19:27 +0000 Subject: [PATCH 04/38] add reverse proxy --- Keter/App.hs | 7 ++-- Keter/Types.hs | 1 - Keter/Types/Common.hs | 52 ++++++++++++++++++++++++++-- Keter/Types/V10.hs | 49 ++------------------------ Network/HTTP/ReverseProxy/Rewrite.hs | 9 ++--- incoming/foo1_0/config/keter.yaml | 7 ++++ 6 files changed, 69 insertions(+), 56 deletions(-) diff --git a/Keter/App.hs b/Keter/App.hs index 8f1e066..e694057 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -173,10 +173,11 @@ withActions asc bconfig f = $ actions0 : map (\host -> Map.singleton host ((PARedirect red, rs), cert)) (Set.toList (redirconfigHosts red)) - loop (Stanza (StanzaReverseProxy rev mid to) rs:stanzas) wacs backs actions0 = - loop stanzas wacs backs actions + loop (Stanza (StanzaReverseProxy rev mid to) rs:stanzas) wacs backs actions0 = do + cert <- loadCert $ reverseUseSSL rev + loop stanzas wacs backs (actions cert) where - actions = Map.insert (CI.mk $ reversingHost rev) ((PAReverseProxy rev mid to, rs), mempty) actions0 + actions cert = Map.insert (CI.mk $ reversingHost rev) ((PAReverseProxy rev mid to, rs), cert) actions0 loop (Stanza (StanzaBackground back) _:stanzas) wacs backs actions = loop stanzas wacs (back:backs) actions diff --git a/Keter/Types.hs b/Keter/Types.hs index 3311ad1..fab86d9 100644 --- a/Keter/Types.hs +++ b/Keter/Types.hs @@ -22,6 +22,5 @@ import Keter.Types.V10 as X , BackgroundConfig (..) , RestartCount (..) , RequiresSecure - , SSLConfig (..) ) import Network.HTTP.ReverseProxy.Rewrite as X (ReverseProxyConfig (..), RewriteRule (..)) diff --git a/Keter/Types/Common.hs b/Keter/Types/Common.hs index 7342e18..3eef890 100644 --- a/Keter/Types/Common.hs +++ b/Keter/Types/Common.hs @@ -1,6 +1,8 @@ {-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE OverloadedStrings #-} + module Keter.Types.Common ( module Keter.Types.Common , FilePath @@ -13,7 +15,9 @@ module Keter.Types.Common ) where import Control.Exception (Exception, SomeException) -import Data.Aeson (Object) +import Data.Aeson (Object, FromJSON, ToJSON, + Value(Bool), (.=), (.!=), (.:?), + withObject, withBool, object) import Data.ByteString (ByteString) import Data.CaseInsensitive (CI, original) import Data.Map (Map) @@ -21,10 +25,13 @@ import Data.Set (Set) import qualified Data.Set as Set import Data.Text (Text, pack, unpack) import Data.Typeable (Typeable) +import Data.Yaml.FilePath +import Data.Vector (Vector) +import qualified Data.Vector as V import qualified Data.Yaml import qualified Language.Haskell.TH.Syntax as TH import System.Exit (ExitCode) -import System.FilePath (takeBaseName) +import System.FilePath (FilePath, takeBaseName) -- | Name of the application. Should just be the basename of the application -- file. @@ -164,3 +171,44 @@ data AppId = AIBuiltin | AINamed !Appname instance Show AppId where show AIBuiltin = "/builtin/" show (AINamed t) = unpack t + +data SSLConfig + = SSLFalse + | SSLTrue + | SSL !FilePath !(Vector FilePath) !FilePath + deriving (Show, Eq, Ord) + +instance ParseYamlFile SSLConfig where + parseYamlFile _ v@(Bool _) = + withBool "ssl" ( \b -> + return (if b then SSLTrue else SSLFalse) ) v + parseYamlFile basedir v = withObject "ssl" ( \o -> do + mcert <- lookupBaseMaybe basedir o "certificate" + mkey <- lookupBaseMaybe basedir o "key" + case (mcert, mkey) of + (Just cert, Just key) -> do + chainCerts <- o .:? "chain-certificates" + >>= maybe (return V.empty) (parseYamlFile basedir) + return $ SSL cert chainCerts key + _ -> return SSLFalse + ) v + +instance ToJSON SSLConfig where + toJSON SSLTrue = Bool True + toJSON SSLFalse = Bool False + toJSON (SSL c cc k) = object [ "certificate" .= c + , "chain-certificates" .= cc + , "key" .= k + ] +instance FromJSON SSLConfig where + parseJSON v@(Bool _) = withBool "ssl" ( \b -> + return (if b then SSLTrue else SSLFalse) ) v + parseJSON v = withObject "ssl" ( \o -> do + mcert <- o .:? "certificate" + mkey <- o .:? "key" + case (mcert, mkey) of + (Just cert, Just key) -> do + chainCerts <- o .:? "chain-certificates" .!= V.empty + return $ SSL cert chainCerts key + _ -> return SSLFalse -- fail "Must provide both certificate and key files" + ) v diff --git a/Keter/Types/V10.hs b/Keter/Types/V10.hs index 6ce3eab..8e4eb31 100644 --- a/Keter/Types/V10.hs +++ b/Keter/Types/V10.hs @@ -6,12 +6,10 @@ module Keter.Types.V10 where import Control.Applicative ((<$>), (<*>), (<|>)) -import Data.Aeson (Object, ToJSON (..)) -import Data.Aeson (FromJSON (..), +import Data.Aeson (FromJSON (..), ToJSON (..), Object, Value (Object, String, Bool), - withObject, withBool, (.!=), (.:), - (.:?)) -import Data.Aeson (Value (Bool), object, (.=)) + withObject, (.!=), (.:), + (.:?), object, (.=)) import qualified Data.CaseInsensitive as CI import Data.Conduit.Network (HostPreference) import Data.Default @@ -352,47 +350,6 @@ instance ToJSON RedirectDest where type IsSecure = Bool -data SSLConfig - = SSLFalse - | SSLTrue - | SSL !F.FilePath !(V.Vector F.FilePath) !F.FilePath - deriving (Show, Eq) - -instance ParseYamlFile SSLConfig where - parseYamlFile _ v@(Bool _) = - withBool "ssl" ( \b -> - return (if b then SSLTrue else SSLFalse) ) v - parseYamlFile basedir v = withObject "ssl" ( \o -> do - mcert <- lookupBaseMaybe basedir o "certificate" - mkey <- lookupBaseMaybe basedir o "key" - case (mcert, mkey) of - (Just cert, Just key) -> do - chainCerts <- o .:? "chain-certificates" - >>= maybe (return V.empty) (parseYamlFile basedir) - return $ SSL cert chainCerts key - _ -> return SSLFalse - ) v - -instance ToJSON SSLConfig where - toJSON SSLTrue = Bool True - toJSON SSLFalse = Bool False - toJSON (SSL c cc k) = object [ "certificate" .= c - , "chain-certificates" .= cc - , "key" .= k - ] -instance FromJSON SSLConfig where - parseJSON v@(Bool _) = withBool "ssl" ( \b -> - return (if b then SSLTrue else SSLFalse) ) v - parseJSON v = withObject "ssl" ( \o -> do - mcert <- o .:? "certificate" - mkey <- o .:? "key" - case (mcert, mkey) of - (Just cert, Just key) -> do - chainCerts <- o .:? "chain-certificates" .!= V.empty - return $ SSL cert chainCerts key - _ -> return SSLFalse -- fail "Must provide both certificate and key files" - ) v - data WebAppConfig port = WebAppConfig { waconfigExec :: !F.FilePath , waconfigArgs :: !(Vector Text) diff --git a/Network/HTTP/ReverseProxy/Rewrite.hs b/Network/HTTP/ReverseProxy/Rewrite.hs index 12999df..95d0bfd 100644 --- a/Network/HTTP/ReverseProxy/Rewrite.hs +++ b/Network/HTTP/ReverseProxy/Rewrite.hs @@ -30,6 +30,7 @@ import qualified Data.CaseInsensitive as CI import Blaze.ByteString.Builder (fromByteString) +import Keter.Types.Common -- Configuration files import Data.Default @@ -126,7 +127,7 @@ mkRequest rpConfig request = , NHC.responseTimeout = reverseTimeout rpConfig #endif , method = Wai.requestMethod request - , secure = reverseUseSSL rpConfig + , secure = SSLFalse /= reverseUseSSL rpConfig , host = encodeUtf8 $ reversedHost rpConfig , port = reversedPort rpConfig , path = Wai.rawPathInfo request @@ -164,7 +165,7 @@ data ReverseProxyConfig = ReverseProxyConfig { reversedHost :: Text , reversedPort :: Int , reversingHost :: Text - , reverseUseSSL :: Bool + , reverseUseSSL :: !SSLConfig , reverseTimeout :: Maybe Int , rewriteResponseRules :: Set RewriteRule , rewriteRequestRules :: Set RewriteRule @@ -175,7 +176,7 @@ instance FromJSON ReverseProxyConfig where <$> o .: "reversed-host" <*> o .: "reversed-port" <*> o .: "reversing-host" - <*> o .:? "ssl" .!= False + <*> o .:? "ssl" .!= SSLFalse <*> o .:? "timeout" .!= Nothing <*> o .:? "rewrite-response" .!= Set.empty <*> o .:? "rewrite-request" .!= Set.empty @@ -197,7 +198,7 @@ instance Default ReverseProxyConfig where { reversedHost = "" , reversedPort = 80 , reversingHost = "" - , reverseUseSSL = False + , reverseUseSSL = SSLFalse , reverseTimeout = Nothing , rewriteResponseRules = Set.empty , rewriteRequestRules = Set.empty diff --git a/incoming/foo1_0/config/keter.yaml b/incoming/foo1_0/config/keter.yaml index e22d7ae..4f8b2ff 100644 --- a/incoming/foo1_0/config/keter.yaml +++ b/incoming/foo1_0/config/keter.yaml @@ -83,6 +83,13 @@ stanzas: # keter : rocks # - headers: # Access-Control-Allow-Origin : "*" + # ssl : true + # ssl: + # key: /opt/keter/etc/cert/hello.key + # certificate: /opt/keter/etc/cert/hello.crt + # chain-certificates: + # - /opt/keter/etc/middle.crt + # - /opt/keter/etc/root.crt - type: redirect hosts: - asome.com From fce6425853ea793033531a3beb11e9fe264ccd96 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 19 Feb 2019 01:35:53 +0000 Subject: [PATCH 05/38] * Builds with `process` 1.6 * add dependency for `tls-session-manager` * bump resolver --- Keter/Main.hs | 10 +++++----- Keter/Proxy.hs | 2 ++ stack.yaml | 5 +++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Keter/Main.hs b/Keter/Main.hs index a8dc353..5dda8b8 100644 --- a/Keter/Main.hs +++ b/Keter/Main.hs @@ -38,7 +38,7 @@ import Data.Text.Encoding (encodeUtf8) import qualified Data.Text.Read import Data.Time (getCurrentTime) import Data.Yaml.FilePath -import qualified Network.HTTP.Conduit as HTTP (conduitManagerSettings, +import qualified Network.HTTP.Conduit as HTTP (tlsManagerSettings, newManager) import Prelude hiding (FilePath, log) import System.Directory (createDirectoryIfMissing, @@ -159,13 +159,13 @@ startWatching kc@KeterConfig {..} appMan log = do _ <- FSN.watchTree wm (fromString incoming) (const True) $ \e -> do e' <- case e of - FSN.Removed fp _ -> do + FSN.Removed fp _ _ -> do log $ WatchedFile "removed" (fromFilePath fp) return $ Left $ fromFilePath fp - FSN.Added fp _ -> do + FSN.Added fp _ _ -> do log $ WatchedFile "added" (fromFilePath fp) return $ Right $ fromFilePath fp - FSN.Modified fp _ -> do + FSN.Modified fp _ _ -> do log $ WatchedFile "modified" (fromFilePath fp) return $ Right $ fromFilePath fp case e' of @@ -208,7 +208,7 @@ listDirectoryTree fp = do startListening :: KeterConfig -> HostMan.HostManager -> IO () startListening KeterConfig {..} hostman = do - manager <- HTTP.newManager HTTP.conduitManagerSettings + manager <- HTTP.newManager HTTP.tlsManagerSettings runAndBlock kconfigListeners $ Proxy.reverseProxy kconfigIpFromHeader -- calculate the number of microseconds since the diff --git a/Keter/Proxy.hs b/Keter/Proxy.hs index 084c6be..074b32e 100644 --- a/Keter/Proxy.hs +++ b/Keter/Proxy.hs @@ -37,6 +37,8 @@ import Network.HTTP.ReverseProxy (ProxyDest (ProxyDest), LocalWaiProxySettings, setLpsTimeBound, waiProxyToSettings, + defaultWaiProxySettings, + defaultLocalWaiProxySettings, wpsSetIpHeader, wpsGetDest) import qualified Network.HTTP.ReverseProxy.Rewrite as Rewrite diff --git a/stack.yaml b/stack.yaml index 308d6ef..6bc9de9 100644 --- a/stack.yaml +++ b/stack.yaml @@ -2,5 +2,6 @@ flags: keter: {} packages: - '.' -extra-deps: [] -resolver: lts-10.1 +resolver: lts-13.8 + +allow-newer: true From 72663af04d4cf795f1cf846c91da30ebcd0598be Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 19 Feb 2019 16:17:40 +0000 Subject: [PATCH 06/38] fix reverse proxy for non http1.1 connections --- Keter/App.hs | 2 +- Network/HTTP/ReverseProxy/Rewrite.hs | 13 +++++++++---- incoming/foo1_0/config/keter.yaml | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Keter/App.hs b/Keter/App.hs index e694057..f25d7ed 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -174,7 +174,7 @@ withActions asc bconfig f = : map (\host -> Map.singleton host ((PARedirect red, rs), cert)) (Set.toList (redirconfigHosts red)) loop (Stanza (StanzaReverseProxy rev mid to) rs:stanzas) wacs backs actions0 = do - cert <- loadCert $ reverseUseSSL rev + cert <- loadCert $ reversingUseSSL rev loop stanzas wacs backs (actions cert) where actions cert = Map.insert (CI.mk $ reversingHost rev) ((PAReverseProxy rev mid to, rs), cert) actions0 diff --git a/Network/HTTP/ReverseProxy/Rewrite.hs b/Network/HTTP/ReverseProxy/Rewrite.hs index 95d0bfd..c0fb5fd 100644 --- a/Network/HTTP/ReverseProxy/Rewrite.hs +++ b/Network/HTTP/ReverseProxy/Rewrite.hs @@ -127,7 +127,7 @@ mkRequest rpConfig request = , NHC.responseTimeout = reverseTimeout rpConfig #endif , method = Wai.requestMethod request - , secure = SSLFalse /= reverseUseSSL rpConfig + , secure = reversedUseSSL rpConfig , host = encodeUtf8 $ reversedHost rpConfig , port = reversedPort rpConfig , path = Wai.rawPathInfo request @@ -140,6 +140,7 @@ mkRequest rpConfig request = , decompress = const False , redirectCount = 0 , cookieJar = Nothing + , requestVersion = Wai.httpVersion request } where reqRuleMap = mkRuleMap $ rewriteRequestRules rpConfig @@ -164,8 +165,9 @@ simpleReverseProxy mgr rpConfig request sendResponse = bracket data ReverseProxyConfig = ReverseProxyConfig { reversedHost :: Text , reversedPort :: Int + , reversedUseSSL :: Bool , reversingHost :: Text - , reverseUseSSL :: !SSLConfig + , reversingUseSSL :: !SSLConfig , reverseTimeout :: Maybe Int , rewriteResponseRules :: Set RewriteRule , rewriteRequestRules :: Set RewriteRule @@ -175,6 +177,7 @@ instance FromJSON ReverseProxyConfig where parseJSON (Object o) = ReverseProxyConfig <$> o .: "reversed-host" <*> o .: "reversed-port" + <*> o .: "reversed-ssl" .!= False <*> o .: "reversing-host" <*> o .:? "ssl" .!= SSLFalse <*> o .:? "timeout" .!= Nothing @@ -186,8 +189,9 @@ instance ToJSON ReverseProxyConfig where toJSON ReverseProxyConfig {..} = object [ "reversed-host" .= reversedHost , "reversed-port" .= reversedPort + , "reversed-ssl" .= reversedUseSSL , "reversing-host" .= reversingHost - , "ssl" .= reverseUseSSL + , "ssl" .= reversingUseSSL , "timeout" .= reverseTimeout , "rewrite-response" .= rewriteResponseRules , "rewrite-request" .= rewriteRequestRules @@ -197,8 +201,9 @@ instance Default ReverseProxyConfig where def = ReverseProxyConfig { reversedHost = "" , reversedPort = 80 + , reversedUseSSL = False , reversingHost = "" - , reverseUseSSL = SSLFalse + , reversingUseSSL = SSLFalse , reverseTimeout = Nothing , rewriteResponseRules = Set.empty , rewriteRequestRules = Set.empty diff --git a/incoming/foo1_0/config/keter.yaml b/incoming/foo1_0/config/keter.yaml index 4f8b2ff..1d5f2bb 100644 --- a/incoming/foo1_0/config/keter.yaml +++ b/incoming/foo1_0/config/keter.yaml @@ -73,6 +73,7 @@ stanzas: - type: reverse-proxy reversed-host: www.yesodweb.com reversed-port: 80 + reversed-ssl: false reversing-host: localhost # connection-time-bound: 0 # middleware: From e34b0e6145147400fa605084c1e82a3ad4a4195d Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 19 Feb 2019 16:37:40 +0000 Subject: [PATCH 07/38] Cleanup --- ChangeLog.md | 5 ++++- Keter/App.hs | 2 +- stack.yaml | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 3f170a5..12ba8e0 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,7 +1,10 @@ ## 1.5 * Builds with `process` 1.6 -* add dependency for `tls-session-manager` +* add dependency for `tls-session-manager`, and it's support +* add support for ssl certs for redirects, static-files, reverse proxies +* fix http2 on reverse proxies +* make it compile with ghc-8.6.3 ## 1.4.3.1 diff --git a/Keter/App.hs b/Keter/App.hs index f25d7ed..55f9a6d 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -133,7 +133,7 @@ withReservations asc aid bconfig f = withActions asc bconfig $ \wacs backs actio withActions :: AppStartConfig -> BundleConfig - -> ([ WebAppConfig Port] -> [BackgroundConfig] -> Map Host (ProxyAction, TLS.Credentials) -> IO a) + -> ([WebAppConfig Port] -> [BackgroundConfig] -> Map Host (ProxyAction, TLS.Credentials) -> IO a) -> IO a withActions asc bconfig f = loop (V.toList $ bconfigStanzas bconfig) [] [] Map.empty diff --git a/stack.yaml b/stack.yaml index 6bc9de9..96d4eb1 100644 --- a/stack.yaml +++ b/stack.yaml @@ -2,6 +2,7 @@ flags: keter: {} packages: - '.' +extra-deps: [] resolver: lts-13.8 allow-newer: true From 9b6ffff69e8ce39975f81151013341d35cb102a1 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Wed, 20 Feb 2019 16:49:53 +0000 Subject: [PATCH 08/38] Make it compile against more resolvers: resolver: lts-13.8 resolver: lts-12.8 resolver: lts-11.8 resolver: lts-10.8 --- Keter/Main.hs | 12 +++++++++--- Keter/Proxy.hs | 14 ++++++++++---- stack.yaml | 3 +++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Keter/Main.hs b/Keter/Main.hs index 5dda8b8..72c5e1e 100644 --- a/Keter/Main.hs +++ b/Keter/Main.hs @@ -152,6 +152,12 @@ getIncoming kc = kconfigDir kc "incoming" isKeter :: FilePath -> Bool isKeter fp = takeExtension fp == ".keter" +#if MIN_VERSION_fsnotify(3,0,0) +#define IGNORE _ +#else +#define IGNORE +#endif + startWatching :: KeterConfig -> AppMan.AppManager -> (LogMessage -> IO ()) -> IO () startWatching kc@KeterConfig {..} appMan log = do -- File system watching @@ -159,13 +165,13 @@ startWatching kc@KeterConfig {..} appMan log = do _ <- FSN.watchTree wm (fromString incoming) (const True) $ \e -> do e' <- case e of - FSN.Removed fp _ _ -> do + FSN.Removed fp _ IGNORE -> do log $ WatchedFile "removed" (fromFilePath fp) return $ Left $ fromFilePath fp - FSN.Added fp _ _ -> do + FSN.Added fp _ IGNORE -> do log $ WatchedFile "added" (fromFilePath fp) return $ Right $ fromFilePath fp - FSN.Modified fp _ _ -> do + FSN.Modified fp _ IGNORE -> do log $ WatchedFile "modified" (fromFilePath fp) return $ Right $ fromFilePath fp case e' of diff --git a/Keter/Proxy.hs b/Keter/Proxy.hs index 074b32e..3751448 100644 --- a/Keter/Proxy.hs +++ b/Keter/Proxy.hs @@ -27,18 +27,21 @@ import qualified Data.Vector as V import Keter.Types import Keter.Types.Middleware import Network.HTTP.Conduit (Manager) + +#if MIN_VERSION_http_reverse_proxy(0,4,2) +import Network.HTTP.ReverseProxy (defaultLocalWaiProxySettings) +#endif + #if MIN_VERSION_http_reverse_proxy(0,6,0) -import Network.HTTP.ReverseProxy (defaultWaiProxySettings, - defaultLocalWaiProxySettings) +import Network.HTTP.ReverseProxy (defaultWaiProxySettings) #endif + import Network.HTTP.ReverseProxy (ProxyDest (ProxyDest), SetIpHeader (..), WaiProxyResponse (..), LocalWaiProxySettings, setLpsTimeBound, waiProxyToSettings, - defaultWaiProxySettings, - defaultLocalWaiProxySettings, wpsSetIpHeader, wpsGetDest) import qualified Network.HTTP.ReverseProxy.Rewrite as Rewrite @@ -59,6 +62,9 @@ import qualified Network.TLS as TLS #if !MIN_VERSION_http_reverse_proxy(0,6,0) defaultWaiProxySettings = def +#endif + +#if !MIN_VERSION_http_reverse_proxy(0,4,2) defaultLocalWaiProxySettings = def #endif diff --git a/stack.yaml b/stack.yaml index 96d4eb1..d2a0456 100644 --- a/stack.yaml +++ b/stack.yaml @@ -3,6 +3,9 @@ flags: packages: - '.' extra-deps: [] +#resolver: lts-10.8 +#resolver: lts-11.8 +#resolver: lts-12.8 resolver: lts-13.8 allow-newer: true From 6ee5ef99ac83498bbd42e8c32e43542325f97b71 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 19 Mar 2019 14:19:32 +0000 Subject: [PATCH 09/38] Fix minimal minimal version handling for `fsnotify` --- Keter/Main.hs | 8 +++++++- keter.cabal | 4 ++-- stack.yaml | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Keter/Main.hs b/Keter/Main.hs index 72c5e1e..24c1345 100644 --- a/Keter/Main.hs +++ b/Keter/Main.hs @@ -152,7 +152,7 @@ getIncoming kc = kconfigDir kc "incoming" isKeter :: FilePath -> Bool isKeter fp = takeExtension fp == ".keter" -#if MIN_VERSION_fsnotify(3,0,0) +#if MIN_VERSION_fsnotify(0,3,0) #define IGNORE _ #else #define IGNORE @@ -174,6 +174,12 @@ startWatching kc@KeterConfig {..} appMan log = do FSN.Modified fp _ IGNORE -> do log $ WatchedFile "modified" (fromFilePath fp) return $ Right $ fromFilePath fp +#if MIN_VERSION_fsnotify(0,3,0) + FSN.Unknown fp _ _ -> do + log $ WatchedFile "unknown" (fromFilePath fp) + return $ Right $ fromFilePath fp +#endif + case e' of Left fp -> when (isKeter fp) $ AppMan.terminateApp appMan $ getAppname fp Right fp -> when (isKeter fp) $ AppMan.addApp appMan $ incoming fp diff --git a/keter.cabal b/keter.cabal index 1cc6be0..d9acfdc 100644 --- a/keter.cabal +++ b/keter.cabal @@ -36,11 +36,11 @@ Library , tar >= 0.4 , template-haskell , blaze-builder >= 0.3 && < 0.5 - , yaml >= 0.8.4 && < 0.9 + , yaml >= 0.8.4 && < 0.12 , unix-compat >= 0.3 && < 0.6 , conduit >= 1.1 , conduit-extra >= 1.1 - , http-reverse-proxy >= 0.4.2 && < 0.5 + , http-reverse-proxy >= 0.4.2 && < 0.7 , unix >= 2.5 , wai-app-static >= 3.1 && < 3.2 , wai >= 3.0 && < 3.3 diff --git a/stack.yaml b/stack.yaml index d2a0456..66789d9 100644 --- a/stack.yaml +++ b/stack.yaml @@ -6,6 +6,7 @@ extra-deps: [] #resolver: lts-10.8 #resolver: lts-11.8 #resolver: lts-12.8 -resolver: lts-13.8 +#resolver: lts-13.8 +resolver: lts-13.13 allow-newer: true From 0087bec7260c4c522e45e47790a4171b89b44cf7 Mon Sep 17 00:00:00 2001 From: idcm Date: Sun, 24 Mar 2019 15:13:23 +0800 Subject: [PATCH 10/38] Update .travis.yml --- .travis.yml | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index a81942c..ca3e090 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,33 +1,19 @@ -language: haskell +sudo: false +language: generic +addons: + apt: + packages: + - libgmp-dev before_install: -env: - #- GHCVER=7.4.2 CABALVER=1.18 - - GHCVER=7.6.3 CABALVER=1.18 - - GHCVER=7.8.4 CABALVER=1.18 - - GHCVER=7.10.1 CABALVER=1.22 - - GHCVER=head CABALVER=head - -matrix: - allow_failures: - - env: GHCVER=head CABALVER=head - -before_install: - - travis_retry sudo add-apt-repository -y ppa:hvr/ghc - - travis_retry sudo apt-get update - - travis_retry sudo apt-get install --force-yes -y cabal-install-$CABALVER ghc-$GHCVER - - export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$HOME/.cabal/bin:$PATH - - cabal --version - -install: - - travis_retry cabal update - - cabal install --only-dependencies --enable-tests +- unset CC +- mkdir -p ~/.local/bin +- export PATH=$HOME/.local/bin:$PATH +- travis_retry curl -L https://www.stackage.org/stack/linux-x86_64 | tar xz --wildcards + --strip-components=1 -C ~/.local/bin '*/stack' script: - - cabal configure --enable-tests -#--enable-library-coverage || cabal configure --enable-tests --enable-coverage - - cabal build - - cabal test - -#after_script: -# - cabal install hpc-coveralls -# - hpc-coveralls --exclude-dir=test test +- stack init --no-terminal +- stack build --no-terminal --test --bench --jobs 1 +cache: + directories: + - "$HOME/.stack" From e54d570b5470e8e6474c12b13958aa4f907f17cb Mon Sep 17 00:00:00 2001 From: idcm Date: Sun, 24 Mar 2019 15:15:07 +0800 Subject: [PATCH 11/38] Delete stack.yaml --- stack.yaml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 stack.yaml diff --git a/stack.yaml b/stack.yaml deleted file mode 100644 index 66789d9..0000000 --- a/stack.yaml +++ /dev/null @@ -1,12 +0,0 @@ -flags: - keter: {} -packages: -- '.' -extra-deps: [] -#resolver: lts-10.8 -#resolver: lts-11.8 -#resolver: lts-12.8 -#resolver: lts-13.8 -resolver: lts-13.13 - -allow-newer: true From 0554b9ef8051f29a7c6052caa748683643bfb908 Mon Sep 17 00:00:00 2001 From: idcm Date: Sun, 24 Mar 2019 15:21:14 +0800 Subject: [PATCH 12/38] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3846527..c72fa9e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ log/ .stack-work/ cabal.sandbox.config *.sublime-* +stack.yaml From 671fc890c5e744e7ae7dfc5f4d71c20c6011051b Mon Sep 17 00:00:00 2001 From: idcm Date: Sun, 24 Mar 2019 15:34:02 +0800 Subject: [PATCH 13/38] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3fa62a6..3a5a97c 100755 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Build Status](https://travis-ci.com/idcm/keter.svg?branch=modernize)](https://travis-ci.com/idcm/keter) Deployment system for web applications, originally intended for hosting Yesod applications. Keter does the following actions for your application: From 0a361a353fa2d7661a8e16125f03280525719c9b Mon Sep 17 00:00:00 2001 From: idcm Date: Sun, 24 Mar 2019 15:34:30 +0800 Subject: [PATCH 14/38] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3a5a97c..70a7336 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ [![Build Status](https://travis-ci.com/idcm/keter.svg?branch=modernize)](https://travis-ci.com/idcm/keter) + + Deployment system for web applications, originally intended for hosting Yesod applications. Keter does the following actions for your application: From 0e412b02e32c93b0a229b39e034142f37f903a99 Mon Sep 17 00:00:00 2001 From: cqs Date: Sun, 24 Mar 2019 17:54:15 +0800 Subject: [PATCH 15/38] clean --- Data/Conduit/Process/Unix.hs | 9 +++++---- Keter/Types/Middleware.hs | 1 - Network/HTTP/ReverseProxy/Rewrite.hs | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Data/Conduit/Process/Unix.hs b/Data/Conduit/Process/Unix.hs index 4ec53a2..528180d 100644 --- a/Data/Conduit/Process/Unix.hs +++ b/Data/Conduit/Process/Unix.hs @@ -33,7 +33,7 @@ import Control.Exception (Exception, SomeException, import Control.Monad (void) import Data.ByteString (ByteString) import qualified Data.ByteString.Char8 as S8 -import Data.Conduit (Source, ($$)) +import Data.Conduit (ConduitT, (.|), runConduit) import Data.Conduit.Binary (sinkHandle, sourceHandle) import qualified Data.Conduit.List as CL import Data.IORef (IORef, newIORef, readIORef, @@ -191,7 +191,7 @@ forkExecuteLog :: ByteString -- ^ command -> [ByteString] -- ^ args -> Maybe [(ByteString, ByteString)] -- ^ environment -> Maybe ByteString -- ^ working directory - -> Maybe (Source IO ByteString) -- ^ stdin + -> Maybe (ConduitT () ByteString IO ()) -- ^ stdin -> (ByteString -> IO ()) -- ^ both stdout and stderr will be sent to this location -> IO ProcessHandle forkExecuteLog cmd args menv mwdir mstdin rlog = bracketOnError @@ -215,6 +215,7 @@ forkExecuteLog cmd args menv mwdir mstdin rlog = bracketOnError , std_err = UseHandle writerH , close_fds = True , create_group = True + , use_process_jobs = False #if MIN_VERSION_process(1, 2, 0) , delegate_ctlc = False #endif @@ -230,10 +231,10 @@ forkExecuteLog cmd args menv mwdir mstdin rlog = bracketOnError } ignoreExceptions $ addAttachMessage pipes ph void $ forkIO $ ignoreExceptions $ - (sourceHandle readerH $$ CL.mapM_ rlog) `finally` hClose readerH + (runConduit $ sourceHandle readerH .| CL.mapM_ rlog) `finally` hClose readerH case (min, mstdin) of (Just h, Just source) -> void $ forkIO $ ignoreExceptions $ - (source $$ sinkHandle h) `finally` hClose h + (runConduit $ source .| sinkHandle h) `finally` hClose h (Nothing, Nothing) -> return () _ -> error $ "Invariant violated: Data.Conduit.Process.Unix.forkExecuteLog" return ph diff --git a/Keter/Types/Middleware.hs b/Keter/Types/Middleware.hs index 6be3d03..d6cc010 100644 --- a/Keter/Types/Middleware.hs +++ b/Keter/Types/Middleware.hs @@ -9,7 +9,6 @@ import Network.Wai import Control.Monad import Control.Arrow ((***)) -import Control.Applicative -- various Middlewares import Network.Wai.Middleware.AcceptOverride (acceptOverride) diff --git a/Network/HTTP/ReverseProxy/Rewrite.hs b/Network/HTTP/ReverseProxy/Rewrite.hs index c0fb5fd..520b247 100644 --- a/Network/HTTP/ReverseProxy/Rewrite.hs +++ b/Network/HTTP/ReverseProxy/Rewrite.hs @@ -42,6 +42,7 @@ import Data.Char (isDigit) -- Reverse proxy apparatus import qualified Network.Wai as Wai +import qualified Network.Wai.Internal as I import Network.HTTP.Client.Conduit import qualified Network.HTTP.Client as NHC import Network.HTTP.Types @@ -135,8 +136,8 @@ mkRequest rpConfig request = , requestHeaders = filterHeaders $ rewriteHeaders reqRuleMap (Wai.requestHeaders request) , requestBody = case Wai.requestBodyLength request of - Wai.ChunkedBody -> RequestBodyStreamChunked ($ Wai.requestBody request) - Wai.KnownLength n -> RequestBodyStream (fromIntegral n) ($ Wai.requestBody request) + Wai.ChunkedBody -> RequestBodyStreamChunked ($ I.getRequestBodyChunk request) + Wai.KnownLength n -> RequestBodyStream (fromIntegral n) ($ I.getRequestBodyChunk request) , decompress = const False , redirectCount = 0 , cookieJar = Nothing From 247b4fc1163e8e18642809cb8a76340cd500b92e Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 13:25:14 +0800 Subject: [PATCH 16/38] Update setup-keter.sh --- setup-keter.sh | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/setup-keter.sh b/setup-keter.sh index f540cb5..6def949 100755 --- a/setup-keter.sh +++ b/setup-keter.sh @@ -1,25 +1,8 @@ #!/bin/bash -set -o errexit -o nounset -o xtrace +mkdir -p /opt/keter/bin +cp ~/.local/bin/keter /opt/keter/bin -# Quick start: -# wget -O - https://raw.github.com/snoyberg/keter/master/setup-keter.sh | bash -ex - -LSB_RELEASE=$(lsb_release -sc) - -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 575159689BEFB442 -echo "deb http://download.fpcomplete.com/ubuntu \"${LSB_RELEASE}\" main"|sudo tee /etc/apt/sources.list.d/fpco.list - -sudo apt-get update -sudo apt-get -y install postgresql stack zlib1g-dev - -stack update -stack setup -stack install keter - -sudo mkdir -p /opt/keter/bin -sudo cp ~/.local/bin/keter /opt/keter/bin - -sudo mkdir -p /opt/keter/etc +mkdir -p /opt/keter/etc cat > /tmp/keter-config.yaml < /tmp/keter.service < Date: Mon, 25 Mar 2019 14:12:37 +0800 Subject: [PATCH 17/38] Update PortPool.hs --- Keter/PortPool.hs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Keter/PortPool.hs b/Keter/PortPool.hs index 9ec1eae..c858155 100644 --- a/Keter/PortPool.hs +++ b/Keter/PortPool.hs @@ -17,6 +17,7 @@ import Control.Concurrent.MVar import Control.Exception import Keter.Types import qualified Network +import Network.Socket import Prelude hiding (log) data PPState = PPState @@ -38,13 +39,13 @@ getPort log (PortPool mstate) = case ppAvail of p:ps -> do let next = PPState ps ppRecycled - res <- try $ Network.listenOn $ Network.PortNumber $ fromIntegral p + res <- try $ listenOn $ fromIntegral p case res of Left (_ :: SomeException) -> do log $ RemovingPort p loop next Right socket -> do - res' <- try $ Network.sClose socket + res' <- try $ close socket case res' of Left e -> do $logEx log e @@ -56,6 +57,22 @@ getPort log (PortPool mstate) = [] -> return (PPState [] id, Left $ toException NoPortsAvailable) ps -> loop $ PPState ps id + listenOn port = do + let hints = defaultHints { + addrFlags = [AI_PASSIVE] + , addrSocketType = Stream + } + addr:_ <- getAddrInfo (Just hints) Nothing (Just port) + bracketOnError + (socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)) + (close) + (\sock -> do + setSocketOption sock ReuseAddr 1 + bind sock (addrAddress addr) + listen sock maxListenQueue + return sock + ) + -- | Return a port to the recycled collection of the pool. Note that recycling -- puts the new ports at the end of the queue (FIFO), so that if an application -- holds onto the port longer than expected, there should be no issues. From 61d5587458189952e1c932d16e2ea41bf6f08a21 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 14:19:01 +0800 Subject: [PATCH 18/38] Update PortPool.hs --- Keter/PortPool.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Keter/PortPool.hs b/Keter/PortPool.hs index c858155..c98fea8 100644 --- a/Keter/PortPool.hs +++ b/Keter/PortPool.hs @@ -39,7 +39,7 @@ getPort log (PortPool mstate) = case ppAvail of p:ps -> do let next = PPState ps ppRecycled - res <- try $ listenOn $ fromIntegral p + res <- try $ listenOn $ show p case res of Left (_ :: SomeException) -> do log $ RemovingPort p From 2e43b970450dd6f77b5f081d81faf92972e74042 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 14:20:00 +0800 Subject: [PATCH 19/38] Update PortPool.hs --- Keter/PortPool.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Keter/PortPool.hs b/Keter/PortPool.hs index c98fea8..01e0c19 100644 --- a/Keter/PortPool.hs +++ b/Keter/PortPool.hs @@ -16,7 +16,6 @@ import Control.Applicative ((<$>)) import Control.Concurrent.MVar import Control.Exception import Keter.Types -import qualified Network import Network.Socket import Prelude hiding (log) From 7dca6ea2084a9169b096c8f28746dcd59c929bc6 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 14:22:01 +0800 Subject: [PATCH 20/38] Update PortPool.hs --- Keter/PortPool.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Keter/PortPool.hs b/Keter/PortPool.hs index 01e0c19..f540f0c 100644 --- a/Keter/PortPool.hs +++ b/Keter/PortPool.hs @@ -43,8 +43,8 @@ getPort log (PortPool mstate) = Left (_ :: SomeException) -> do log $ RemovingPort p loop next - Right socket -> do - res' <- try $ close socket + Right socket' -> do + res' <- try $ close socket' case res' of Left e -> do $logEx log e From c17679fd7762f07b7fe1f85c25c79cb83699583b Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:01:39 +0800 Subject: [PATCH 21/38] Update App.hs --- Keter/App.hs | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/Keter/App.hs b/Keter/App.hs index 55f9a6d..a143e13 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -46,7 +46,7 @@ import Keter.Types import qualified Network import Prelude hiding (FilePath) import System.Environment (getEnvironment) -import System.IO (hClose) +import System.IO (hClose, ioError, userError) import System.Posix.Files (fileAccess) import System.Posix.Types (EpochTime, GroupID, UserID) import System.Timeout (timeout) @@ -338,12 +338,37 @@ ensureAlive RunningWebApp {..} = do where testApp' = do threadDelay $ 2 * 1000 * 1000 - eres <- try $ Network.connectTo "127.0.0.1" $ Network.PortNumber $ fromIntegral port + eres <- try $ connectTo "127.0.0.1" $ show port case eres of Left (_ :: IOException) -> testApp' Right handle -> do hClose handle return True + connectTo hos pr = do + let hints = defaultHints { addrFlags = [AI_ADDRCONFIG] + , addrSocketType = Stream } + addrs <- getAddrInfo (Just hints) (Just host) (Just serv) + firstSuccessful $ map tryToConnect addrs + where + tryToConnect addr = + bracketOnError + (socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)) + (close) -- only done if there's an error + (\sock -> do + connect sock (addrAddress addr) + socketToHandle sock ReadWriteMode + ) + firstSuccessful = go Nothing + where + go _ (p:ps) = do + r <- tryIO p + case r of + Right x -> return x + Left e -> go (Just e) ps + -- All operations failed, throw error if one exists + go Nothing [] = ioError $ userError $ "connectTo firstSuccessful: empty list" + go (Just e) [] = throwIO e + withBackgroundApps :: AppStartConfig -> AppId From 2460da0bcec2ff54efa46dcab6fc2280772e5ef5 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:03:29 +0800 Subject: [PATCH 22/38] Update App.hs --- Keter/App.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Keter/App.hs b/Keter/App.hs index a143e13..b1169b7 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -46,7 +46,7 @@ import Keter.Types import qualified Network import Prelude hiding (FilePath) import System.Environment (getEnvironment) -import System.IO (hClose, ioError, userError) +import System.IO (hClose) import System.Posix.Files (fileAccess) import System.Posix.Types (EpochTime, GroupID, UserID) import System.Timeout (timeout) From bba765f5504d6698e5fb2cebd08934d12eb81ec3 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:04:24 +0800 Subject: [PATCH 23/38] Update App.hs --- Keter/App.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Keter/App.hs b/Keter/App.hs index b1169b7..d3b5b5e 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -44,6 +44,7 @@ import Keter.HostManager hiding (start) import Keter.PortPool (PortPool, getPort, releasePort) import Keter.Types import qualified Network +import Network.Socket import Prelude hiding (FilePath) import System.Environment (getEnvironment) import System.IO (hClose) From db199ac8d8056cb5bb16b8d8c4b877a3f888f881 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:06:00 +0800 Subject: [PATCH 24/38] Update App.hs --- Keter/App.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Keter/App.hs b/Keter/App.hs index d3b5b5e..249c798 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -345,7 +345,7 @@ ensureAlive RunningWebApp {..} = do Right handle -> do hClose handle return True - connectTo hos pr = do + connectTo host serv = do let hints = defaultHints { addrFlags = [AI_ADDRCONFIG] , addrSocketType = Stream } addrs <- getAddrInfo (Just hints) (Just host) (Just serv) From 98c4c78285d1404d9a6986007c357739fc1f86be Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:08:19 +0800 Subject: [PATCH 25/38] Update App.hs --- Keter/App.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Keter/App.hs b/Keter/App.hs index 249c798..c0a1eed 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -47,7 +47,7 @@ import qualified Network import Network.Socket import Prelude hiding (FilePath) import System.Environment (getEnvironment) -import System.IO (hClose) +import System.IO (hClose, ReadWriteMode) import System.Posix.Files (fileAccess) import System.Posix.Types (EpochTime, GroupID, UserID) import System.Timeout (timeout) From 439fdb76905fca01ccb794beaaaed3d7812fecbc Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:09:01 +0800 Subject: [PATCH 26/38] Update App.hs --- Keter/App.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Keter/App.hs b/Keter/App.hs index c0a1eed..f4193a8 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -47,7 +47,7 @@ import qualified Network import Network.Socket import Prelude hiding (FilePath) import System.Environment (getEnvironment) -import System.IO (hClose, ReadWriteMode) +import System.IO (hClose, IOMode(..)) import System.Posix.Files (fileAccess) import System.Posix.Types (EpochTime, GroupID, UserID) import System.Timeout (timeout) From b9e5c0953a4dc19e38756e5719bf7bba3459300b Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:12:45 +0800 Subject: [PATCH 27/38] Update App.hs --- Keter/App.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Keter/App.hs b/Keter/App.hs index f4193a8..ef40ec5 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -369,6 +369,8 @@ ensureAlive RunningWebApp {..} = do -- All operations failed, throw error if one exists go Nothing [] = ioError $ userError $ "connectTo firstSuccessful: empty list" go (Just e) [] = throwIO e + tryIO :: IO a -> IO (Either IOException a) + tryIO m = catchIO (liftM Right m) (return . Left) withBackgroundApps :: AppStartConfig From 19682557d14e1f863d48ae211becece61e9e4da0 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:15:57 +0800 Subject: [PATCH 28/38] Update App.hs --- Keter/App.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Keter/App.hs b/Keter/App.hs index ef40ec5..22be8c1 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -18,8 +18,8 @@ import Control.Arrow ((***)) import Control.Concurrent (forkIO, threadDelay) import Control.Concurrent.STM import Control.Exception (IOException, bracketOnError, - throwIO, try) -import Control.Monad (void, when) + throwIO, try, catch) +import Control.Monad (void, when, liftM) import qualified Data.CaseInsensitive as CI import Data.Conduit.LogFile (RotatingLog) import qualified Data.Conduit.LogFile as LogFile @@ -370,7 +370,7 @@ ensureAlive RunningWebApp {..} = do go Nothing [] = ioError $ userError $ "connectTo firstSuccessful: empty list" go (Just e) [] = throwIO e tryIO :: IO a -> IO (Either IOException a) - tryIO m = catchIO (liftM Right m) (return . Left) + tryIO m = catch (liftM Right m) (return . Left) withBackgroundApps :: AppStartConfig From 9e7a933dd637bbbf69856ab77ee885eac81f1df3 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:16:44 +0800 Subject: [PATCH 29/38] Update App.hs --- Keter/App.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Keter/App.hs b/Keter/App.hs index 22be8c1..93f9249 100644 --- a/Keter/App.hs +++ b/Keter/App.hs @@ -43,7 +43,6 @@ import System.Directory (canonicalizePath, doesFileExist, import Keter.HostManager hiding (start) import Keter.PortPool (PortPool, getPort, releasePort) import Keter.Types -import qualified Network import Network.Socket import Prelude hiding (FilePath) import System.Environment (getEnvironment) From 5f3bcee681271d7d05a53b795b582fd0063f171e Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:48:08 +0800 Subject: [PATCH 30/38] Update LabelMapSpec.hs --- test/LabelMapSpec.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/LabelMapSpec.hs b/test/LabelMapSpec.hs index a884b42..89ea188 100644 --- a/test/LabelMapSpec.hs +++ b/test/LabelMapSpec.hs @@ -3,7 +3,7 @@ module LabelMapSpec where -import qualified Data.IORef as I +-- import qualified Data.IORef as I import Test.Hspec import Test.HUnit From a26abef2d2d0f141a1e45025666dae971b7ce831 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:51:06 +0800 Subject: [PATCH 31/38] Update LabelMapSpec.hs --- test/LabelMapSpec.hs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/LabelMapSpec.hs b/test/LabelMapSpec.hs index 89ea188..4261a7f 100644 --- a/test/LabelMapSpec.hs +++ b/test/LabelMapSpec.hs @@ -46,13 +46,13 @@ caseSubdomainIntegrity = do test3a = LM.delete "someapp.com" test2 test3b = LM.insert "api.someapp.com" () test0 -- case from the bug report - let test3 = LM.insert "ipa.someapp.com" () test2 - test4 = LM.insert "bla.api.someapp.com" () test3 - test5 = LM.delete "someapp.com" test4 - test6 = LM.delete "ipa.someapp.com" test5 - test7 = LM.delete "api.someapp.com" test6 - test8 = LM.delete "bla.api.someapp.com" test7 - test9 = LM.delete "bla.api.someapp.com" test4 + let -- test3 = LM.insert "ipa.someapp.com" () test2 + -- test4 = LM.insert "bla.api.someapp.com" () test3 + -- test5 = LM.delete "someapp.com" test4 + -- test6 = LM.delete "ipa.someapp.com" test5 + -- test7 = LM.delete "api.someapp.com" test6 + -- test8 = LM.delete "bla.api.someapp.com" test7 + -- test9 = LM.delete "bla.api.someapp.com" test4 msg = "Subdomains inserted and deleted between bundles" print test3a From 4dd6e528cda4baaf54c49346ea592a87fdf5e0e2 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 15:58:02 +0800 Subject: [PATCH 32/38] Update keter.cabal --- keter.cabal | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keter.cabal b/keter.cabal index d9acfdc..4095594 100644 --- a/keter.cabal +++ b/keter.cabal @@ -93,7 +93,8 @@ Library Data.Conduit.Process.Unix ghc-options: -Wall c-sources: cbits/process-tracker.c - + other-modules: LabelMapSpec + Executable keter Main-is: keter.hs hs-source-dirs: main From 008e16ff0c5ed0f63ffcdc523c560a7cb89356e0 Mon Sep 17 00:00:00 2001 From: idcm Date: Mon, 25 Mar 2019 16:03:37 +0800 Subject: [PATCH 33/38] Update keter.cabal --- keter.cabal | 1 - 1 file changed, 1 deletion(-) diff --git a/keter.cabal b/keter.cabal index 4095594..77296fc 100644 --- a/keter.cabal +++ b/keter.cabal @@ -93,7 +93,6 @@ Library Data.Conduit.Process.Unix ghc-options: -Wall c-sources: cbits/process-tracker.c - other-modules: LabelMapSpec Executable keter Main-is: keter.hs From 2cd49d4ec690a70ae12de4eb39c590695d5ba355 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 25 Feb 2020 11:24:47 +0000 Subject: [PATCH 34/38] modernize travis --- .travis.yml | 277 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 265 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index ca3e090..7b8a3e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,272 @@ +# This is the complex Travis configuration, which is intended for use +# on open source libraries which need compatibility across multiple GHC +# versions, must work with cabal-install, and should be +# cross-platform. For more information and other options, see: +# +# https://docs.haskellstack.org/en/stable/travis_ci/ +# +# Copy these contents into the root directory of your Github project in a file +# named .travis.yml + +# Use new container infrastructure to enable caching sudo: false + +# Do not choose a language; we provide our own build tools. language: generic -addons: - apt: - packages: - - libgmp-dev + +# Caching so the next build will be fast too. +cache: + directories: + - $HOME/.ghc + - $HOME/.cabal + - $HOME/.stack + - $TRAVIS_BUILD_DIR/.stack-work + +# The different configurations we want to test. We have BUILD=cabal which uses +# cabal-install, and BUILD=stack which uses Stack. More documentation on each +# of those below. +# +# We set the compiler values here to tell Travis to use a different +# cache file per set of arguments. +# +# If you need to have different apt packages for each combination in the +# matrix, you can use a line such as: +# addons: {apt: {packages: [libfcgi-dev,libgmp-dev]}} +matrix: + include: + # We grab the appropriate GHC and cabal-install versions from hvr's PPA. See: + # https://github.com/hvr/multi-ghc-travis + #- env: BUILD=cabal GHCVER=7.0.4 CABALVER=1.16 HAPPYVER=1.19.5 ALEXVER=3.1.7 + # compiler: ": #GHC 7.0.4" + # addons: {apt: {packages: [cabal-install-1.16,ghc-7.0.4,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + #- env: BUILD=cabal GHCVER=7.2.2 CABALVER=1.16 HAPPYVER=1.19.5 ALEXVER=3.1.7 + # compiler: ": #GHC 7.2.2" + # addons: {apt: {packages: [cabal-install-1.16,ghc-7.2.2,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + #- env: BUILD=cabal GHCVER=7.4.2 CABALVER=1.16 HAPPYVER=1.19.5 ALEXVER=3.1.7 + # compiler: ": #GHC 7.4.2" + # addons: {apt: {packages: [cabal-install-1.16,ghc-7.4.2,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + #- env: BUILD=cabal GHCVER=7.6.3 CABALVER=1.16 HAPPYVER=1.19.5 ALEXVER=3.1.7 + # compiler: ": #GHC 7.6.3" + # addons: {apt: {packages: [cabal-install-1.16,ghc-7.6.3,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + #- env: BUILD=cabal GHCVER=7.8.4 CABALVER=1.18 HAPPYVER=1.19.5 ALEXVER=3.1.7 + # compiler: ": #GHC 7.8.4" + # addons: {apt: {packages: [cabal-install-1.18,ghc-7.8.4,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + #- env: BUILD=cabal GHCVER=7.10.3 CABALVER=1.22 HAPPYVER=1.19.5 ALEXVER=3.1.7 + # compiler: ": #GHC 7.10.3" + # addons: {apt: {packages: [cabal-install-1.22,ghc-7.10.3,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + - env: BUILD=cabal GHCVER=8.0.2 CABALVER=1.24 HAPPYVER=1.19.5 ALEXVER=3.1.7 + compiler: ": #GHC 8.0.2" + addons: {apt: {packages: [cabal-install-1.24,ghc-8.0.2,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + - env: BUILD=cabal GHCVER=8.2.2 CABALVER=2.0 HAPPYVER=1.19.5 ALEXVER=3.1.7 + compiler: ": #GHC 8.2.2" + addons: {apt: {packages: [cabal-install-2.0,ghc-8.2.2,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + - env: BUILD=cabal GHCVER=8.4.4 CABALVER=2.2 HAPPYVER=1.19.5 ALEXVER=3.1.7 + compiler: ": #GHC 8.4.4" + addons: {apt: {packages: [cabal-install-2.2,ghc-8.4.4,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + - env: BUILD=cabal GHCVER=8.6.5 CABALVER=2.4 HAPPYVER=1.19.5 ALEXVER=3.1.7 + compiler: ": #GHC 8.6.5" + addons: {apt: {packages: [cabal-install-2.4,ghc-8.6.5,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + + # Build with the newest GHC and cabal-install. This is an accepted failure, + # see below. + - env: BUILD=cabal GHCVER=head CABALVER=head HAPPYVER=1.19.5 ALEXVER=3.1.7 + compiler: ": #GHC HEAD" + addons: {apt: {packages: [cabal-install-head,ghc-head,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + + # The Stack builds. We can pass in arbitrary Stack arguments via the ARGS + # variable, such as using --stack-yaml to point to a different file. + - env: BUILD=stack ARGS="" + compiler: ": #stack default" + addons: {apt: {packages: [libgmp-dev]}} + + #- env: BUILD=stack ARGS="--resolver lts-2" + # compiler: ": #stack 7.8.4" + # addons: {apt: {packages: [libgmp-dev]}} + + #- env: BUILD=stack ARGS="--resolver lts-3" + # compiler: ": #stack 7.10.2" + # addons: {apt: {packages: [libgmp-dev]}} + + #- env: BUILD=stack ARGS="--resolver lts-6" + # compiler: ": #stack 7.10.3" + # addons: {apt: {packages: [libgmp-dev]}} + + #- env: BUILD=stack ARGS="--resolver lts-7" + # compiler: ": #stack 8.0.1" + # addons: {apt: {packages: [libgmp-dev]}} + + - env: BUILD=stack ARGS="--resolver lts-9" + compiler: ": #stack 8.0.2" + addons: {apt: {packages: [libgmp-dev]}} + + - env: BUILD=stack ARGS="--resolver lts-11" + compiler: ": #stack 8.2.2" + addons: {apt: {packages: [libgmp-dev]}} + + - env: BUILD=stack ARGS="--resolver lts-12" + compiler: ": #stack 8.4.4" + addons: {apt: {packages: [libgmp-dev]}} + + - env: BUILD=stack ARGS="--resolver lts-13" + compiler: ": #stack 8.6.4" + addons: {apt: {packages: [libgmp-dev]}} + + - env: BUILD=stack ARGS="--resolver lts-14" + compiler: ": #stack 8.6.5" + addons: {apt: {packages: [libgmp-dev]}} + + - env: BUILD=stack ARGS="--resolver lts-15" + compiler: ": #stack 8.8.2" + addons: {apt: {packages: [libgmp-dev]}} + + # Nightly builds are allowed to fail + - env: BUILD=stack ARGS="--resolver nightly" + compiler: ": #stack nightly" + addons: {apt: {packages: [libgmp-dev]}} + + # Build on macOS in addition to Linux + - env: BUILD=stack ARGS="" + compiler: ": #stack default osx" + os: osx + + # Travis includes an macOS which is incompatible with GHC 7.8.4 + #- env: BUILD=stack ARGS="--resolver lts-2" + # compiler: ": #stack 7.8.4 osx" + # os: osx + + #- env: BUILD=stack ARGS="--resolver lts-3" + # compiler: ": #stack 7.10.2 osx" + # os: osx + + #- env: BUILD=stack ARGS="--resolver lts-6" + # compiler: ": #stack 7.10.3 osx" + # os: osx + + #- env: BUILD=stack ARGS="--resolver lts-7" + # compiler: ": #stack 8.0.1 osx" + # os: osx + + - env: BUILD=stack ARGS="--resolver lts-9" + compiler: ": #stack 8.0.2 osx" + os: osx + + - env: BUILD=stack ARGS="--resolver lts-11" + compiler: ": #stack 8.2.2 osx" + os: osx + + - env: BUILD=stack ARGS="--resolver lts-12" + compiler: ": #stack 8.4.4 osx" + os: osx + + - env: BUILD=stack ARGS="--resolver lts-13" + compiler: ": #stack 8.6.4 osx" + os: osx + + - env: BUILD=stack ARGS="--resolver lts-14" + compiler: ": #stack 8.6.5 osx" + os: osx + + - env: BUILD=stack ARGS="--resolver lts-15" + compiler: ": #stack 8.8.2 osx" + os: osx + + - env: BUILD=stack ARGS="--resolver nightly" + compiler: ": #stack nightly osx" + os: osx + + allow_failures: + - env: BUILD=cabal GHCVER=head CABALVER=head HAPPYVER=1.19.5 ALEXVER=3.1.7 + - env: BUILD=stack ARGS="--resolver nightly" + before_install: +# Using compiler above sets CC to an invalid value, so unset it - unset CC + +# We want to always allow newer versions of packages when building on GHC HEAD +- CABALARGS="" +- if [ "x$GHCVER" = "xhead" ]; then CABALARGS=--allow-newer; fi + +# Download and unpack the stack executable +- export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$HOME/.local/bin:/opt/alex/$ALEXVER/bin:/opt/happy/$HAPPYVER/bin:$HOME/.cabal/bin:$PATH - mkdir -p ~/.local/bin -- export PATH=$HOME/.local/bin:$PATH -- travis_retry curl -L https://www.stackage.org/stack/linux-x86_64 | tar xz --wildcards - --strip-components=1 -C ~/.local/bin '*/stack' +- | + if [ `uname` = "Darwin" ] + then + travis_retry curl --insecure -L https://get.haskellstack.org/stable/osx-x86_64.tar.gz | tar xz --strip-components=1 --include '*/stack' -C ~/.local/bin + else + travis_retry curl -L https://get.haskellstack.org/stable/linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C ~/.local/bin '*/stack' + fi + + # Use the more reliable S3 mirror of Hackage + mkdir -p $HOME/.cabal + echo 'remote-repo: hackage.haskell.org:http://hackage.fpcomplete.com/' > $HOME/.cabal/config + echo 'remote-repo-cache: $HOME/.cabal/packages' >> $HOME/.cabal/config + + +install: +- echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]" +- if [ -f configure.ac ]; then autoreconf -i; fi +- | + set -ex + case "$BUILD" in + stack) + # Add in extra-deps for older snapshots, as necessary + # + # This is disabled by default, as relying on the solver like this can + # make builds unreliable. Instead, if you have this situation, it's + # recommended that you maintain multiple stack-lts-X.yaml files. + + #stack --no-terminal --install-ghc $ARGS test --bench --dry-run || ( \ + # stack --no-terminal $ARGS build cabal-install && \ + # stack --no-terminal $ARGS solver --update-config) + + # Build the dependencies + stack --no-terminal --install-ghc $ARGS test --bench --only-dependencies + ;; + cabal) + cabal --version + travis_retry cabal update + + # Get the list of packages from the stack.yaml file. Note that + # this will also implicitly run hpack as necessary to generate + # the .cabal files needed by cabal-install. + PACKAGES=$(stack --install-ghc query locals | grep '^ *path' | sed 's@^ *path:@@') + + cabal install --only-dependencies --enable-tests --enable-benchmarks --force-reinstalls --ghc-options=-O0 --reorder-goals --max-backjumps=-1 $CABALARGS $PACKAGES + ;; + esac + set +ex script: -- stack init --no-terminal -- stack build --no-terminal --test --bench --jobs 1 -cache: - directories: - - "$HOME/.stack" +- | + set -ex + case "$BUILD" in + stack) + stack --no-terminal $ARGS test --bench --no-run-benchmarks --haddock --no-haddock-deps + ;; + cabal) + cabal install --enable-tests --enable-benchmarks --force-reinstalls --ghc-options=-O0 --reorder-goals --max-backjumps=-1 $CABALARGS $PACKAGES + + ORIGDIR=$(pwd) + for dir in $PACKAGES + do + cd $dir + cabal check || [ "$CABALVER" == "1.16" ] + cabal sdist + PKGVER=$(cabal info . | awk '{print $2;exit}') + SRC_TGZ=$PKGVER.tar.gz + cd dist + tar zxfv "$SRC_TGZ" + cd "$PKGVER" + cabal configure --enable-tests --ghc-options -O0 + cabal build + if [ "$CABALVER" = "1.16" ] || [ "$CABALVER" = "1.18" ]; then + cabal test + else + cabal test --show-details=streaming --log=/dev/stdout + fi + cd $ORIGDIR + done + ;; + esac + set +ex From fd84c8aff4134d276f8f23530a818a0017de1e0b Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 25 Feb 2020 11:50:30 +0000 Subject: [PATCH 35/38] add stack.yaml --- .gitignore | 2 +- .travis.yml | 12 ++++++------ stack.yaml | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 stack.yaml diff --git a/.gitignore b/.gitignore index c72fa9e..514b631 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,4 @@ log/ .stack-work/ cabal.sandbox.config *.sublime-* -stack.yaml +#stack.yaml diff --git a/.travis.yml b/.travis.yml index 7b8a3e2..9c13d47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,9 +54,9 @@ matrix: #- env: BUILD=cabal GHCVER=7.10.3 CABALVER=1.22 HAPPYVER=1.19.5 ALEXVER=3.1.7 # compiler: ": #GHC 7.10.3" # addons: {apt: {packages: [cabal-install-1.22,ghc-7.10.3,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} - - env: BUILD=cabal GHCVER=8.0.2 CABALVER=1.24 HAPPYVER=1.19.5 ALEXVER=3.1.7 - compiler: ": #GHC 8.0.2" - addons: {apt: {packages: [cabal-install-1.24,ghc-8.0.2,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} + #- env: BUILD=cabal GHCVER=8.0.2 CABALVER=1.24 HAPPYVER=1.19.5 ALEXVER=3.1.7 + # compiler: ": #GHC 8.0.2" + # addons: {apt: {packages: [cabal-install-1.24,ghc-8.0.2,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} - env: BUILD=cabal GHCVER=8.2.2 CABALVER=2.0 HAPPYVER=1.19.5 ALEXVER=3.1.7 compiler: ": #GHC 8.2.2" addons: {apt: {packages: [cabal-install-2.0,ghc-8.2.2,happy-1.19.5,alex-3.1.7], sources: [hvr-ghc]}} @@ -95,9 +95,9 @@ matrix: # compiler: ": #stack 8.0.1" # addons: {apt: {packages: [libgmp-dev]}} - - env: BUILD=stack ARGS="--resolver lts-9" - compiler: ": #stack 8.0.2" - addons: {apt: {packages: [libgmp-dev]}} + #- env: BUILD=stack ARGS="--resolver lts-9" + # compiler: ": #stack 8.0.2" + # addons: {apt: {packages: [libgmp-dev]}} - env: BUILD=stack ARGS="--resolver lts-11" compiler: ": #stack 8.2.2" diff --git a/stack.yaml b/stack.yaml new file mode 100644 index 0000000..9b05b12 --- /dev/null +++ b/stack.yaml @@ -0,0 +1,3 @@ +resolver: lts-15.1 +packages: +- . From d79ebcc418a0b71d7d7765245d999328a6678f37 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 25 Feb 2020 12:21:31 +0000 Subject: [PATCH 36/38] do not support wai prior wai-3.2.2 --- .travis.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c13d47..d024a0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,13 +99,13 @@ matrix: # compiler: ": #stack 8.0.2" # addons: {apt: {packages: [libgmp-dev]}} - - env: BUILD=stack ARGS="--resolver lts-11" - compiler: ": #stack 8.2.2" - addons: {apt: {packages: [libgmp-dev]}} + #- env: BUILD=stack ARGS="--resolver lts-11" + # compiler: ": #stack 8.2.2" + # addons: {apt: {packages: [libgmp-dev]}} - - env: BUILD=stack ARGS="--resolver lts-12" - compiler: ": #stack 8.4.4" - addons: {apt: {packages: [libgmp-dev]}} + #- env: BUILD=stack ARGS="--resolver lts-12" + # compiler: ": #stack 8.4.4" + # addons: {apt: {packages: [libgmp-dev]}} - env: BUILD=stack ARGS="--resolver lts-13" compiler: ": #stack 8.6.4" @@ -146,17 +146,17 @@ matrix: # compiler: ": #stack 8.0.1 osx" # os: osx - - env: BUILD=stack ARGS="--resolver lts-9" - compiler: ": #stack 8.0.2 osx" - os: osx + #- env: BUILD=stack ARGS="--resolver lts-9" + # compiler: ": #stack 8.0.2 osx" + # os: osx - - env: BUILD=stack ARGS="--resolver lts-11" - compiler: ": #stack 8.2.2 osx" - os: osx + # - env: BUILD=stack ARGS="--resolver lts-11" + # compiler: ": #stack 8.2.2 osx" + # os: osx - - env: BUILD=stack ARGS="--resolver lts-12" - compiler: ": #stack 8.4.4 osx" - os: osx + # - env: BUILD=stack ARGS="--resolver lts-12" + # compiler: ": #stack 8.4.4 osx" + # os: osx - env: BUILD=stack ARGS="--resolver lts-13" compiler: ": #stack 8.6.4 osx" From 817dc4bbe4282c73d2b9cf85fdccdbcd71209ad7 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 25 Feb 2020 13:18:21 +0000 Subject: [PATCH 37/38] re-enable support for wai prior wai-3.2.2 --- .travis.yml | 34 ++++++++++++++-------------- Network/HTTP/ReverseProxy/Rewrite.hs | 12 +++++++--- keter.cabal | 4 ++-- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index d024a0d..02a8ef4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,17 +95,21 @@ matrix: # compiler: ": #stack 8.0.1" # addons: {apt: {packages: [libgmp-dev]}} - #- env: BUILD=stack ARGS="--resolver lts-9" + # - env: BUILD=stack ARGS="--resolver lts-9" # compiler: ": #stack 8.0.2" # addons: {apt: {packages: [libgmp-dev]}} - #- env: BUILD=stack ARGS="--resolver lts-11" - # compiler: ": #stack 8.2.2" + # - env: BUILD=stack ARGS="--resolver lts-10" + # compiler: ": #stack 8.2.2" # addons: {apt: {packages: [libgmp-dev]}} - #- env: BUILD=stack ARGS="--resolver lts-12" - # compiler: ": #stack 8.4.4" - # addons: {apt: {packages: [libgmp-dev]}} + - env: BUILD=stack ARGS="--resolver lts-11" + compiler: ": #stack 8.2.2" + addons: {apt: {packages: [libgmp-dev]}} + + - env: BUILD=stack ARGS="--resolver lts-12" + compiler: ": #stack 8.4.4" + addons: {apt: {packages: [libgmp-dev]}} - env: BUILD=stack ARGS="--resolver lts-13" compiler: ": #stack 8.6.4" @@ -146,17 +150,13 @@ matrix: # compiler: ": #stack 8.0.1 osx" # os: osx - #- env: BUILD=stack ARGS="--resolver lts-9" - # compiler: ": #stack 8.0.2 osx" - # os: osx - - # - env: BUILD=stack ARGS="--resolver lts-11" - # compiler: ": #stack 8.2.2 osx" - # os: osx + - env: BUILD=stack ARGS="--resolver lts-11" + compiler: ": #stack 8.2.2 osx" + os: osx - # - env: BUILD=stack ARGS="--resolver lts-12" - # compiler: ": #stack 8.4.4 osx" - # os: osx + - env: BUILD=stack ARGS="--resolver lts-12" + compiler: ": #stack 8.4.4 osx" + os: osx - env: BUILD=stack ARGS="--resolver lts-13" compiler: ": #stack 8.6.4 osx" @@ -201,7 +201,7 @@ before_install: mkdir -p $HOME/.cabal echo 'remote-repo: hackage.haskell.org:http://hackage.fpcomplete.com/' > $HOME/.cabal/config echo 'remote-repo-cache: $HOME/.cabal/packages' >> $HOME/.cabal/config - + stack upgrade --git install: - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]" diff --git a/Network/HTTP/ReverseProxy/Rewrite.hs b/Network/HTTP/ReverseProxy/Rewrite.hs index 520b247..a7038a4 100644 --- a/Network/HTTP/ReverseProxy/Rewrite.hs +++ b/Network/HTTP/ReverseProxy/Rewrite.hs @@ -42,11 +42,17 @@ import Data.Char (isDigit) -- Reverse proxy apparatus import qualified Network.Wai as Wai -import qualified Network.Wai.Internal as I import Network.HTTP.Client.Conduit import qualified Network.HTTP.Client as NHC import Network.HTTP.Types +getRequestBodyChunk :: Wai.Request -> IO ByteString +#if MIN_VERSION_wai(3, 2, 2) +getRequestBodyChunk = Wai.getRequestBodyChunk +#else +getRequestBodyChunk = Wai.requestBody +#endif + data RPEntry = RPEntry { config :: ReverseProxyConfig , httpManager :: Manager @@ -136,8 +142,8 @@ mkRequest rpConfig request = , requestHeaders = filterHeaders $ rewriteHeaders reqRuleMap (Wai.requestHeaders request) , requestBody = case Wai.requestBodyLength request of - Wai.ChunkedBody -> RequestBodyStreamChunked ($ I.getRequestBodyChunk request) - Wai.KnownLength n -> RequestBodyStream (fromIntegral n) ($ I.getRequestBodyChunk request) + Wai.ChunkedBody -> RequestBodyStreamChunked ($ getRequestBodyChunk request) + Wai.KnownLength n -> RequestBodyStream (fromIntegral n) ($ getRequestBodyChunk request) , decompress = const False , redirectCount = 0 , cookieJar = Nothing diff --git a/keter.cabal b/keter.cabal index 77296fc..2ad9efd 100644 --- a/keter.cabal +++ b/keter.cabal @@ -38,8 +38,8 @@ Library , blaze-builder >= 0.3 && < 0.5 , yaml >= 0.8.4 && < 0.12 , unix-compat >= 0.3 && < 0.6 - , conduit >= 1.1 - , conduit-extra >= 1.1 + , conduit >= 1.3 + , conduit-extra >= 1.3 , http-reverse-proxy >= 0.4.2 && < 0.7 , unix >= 2.5 , wai-app-static >= 3.1 && < 3.2 From db6a104aa252f49f98ee79a1d88b3fb343f4f635 Mon Sep 17 00:00:00 2001 From: Marcin Tolysz Date: Tue, 25 Feb 2020 13:48:27 +0000 Subject: [PATCH 38/38] back to stable stack --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 02a8ef4..85ebf25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -201,7 +201,6 @@ before_install: mkdir -p $HOME/.cabal echo 'remote-repo: hackage.haskell.org:http://hackage.fpcomplete.com/' > $HOME/.cabal/config echo 'remote-repo-cache: $HOME/.cabal/packages' >> $HOME/.cabal/config - stack upgrade --git install: - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]"