Browse Source

reorganizating and adding goatcounter stuff

sni-wip
forest 9 months ago
parent
commit
c166942f78
22 changed files with 411 additions and 19 deletions
  1. +19
    -17
      build-docker.sh
  2. +23
    -0
      caddy/config/config.json
  3. +3
    -0
      docker-compose.yml
  4. +0
    -0
      dockerbuild_ghost/Dockerfile
  5. +0
    -0
      dockerbuild_ghost/docker-entrypoint.sh
  6. +0
    -0
      dockerbuild_gitea/build-docker.sh
  7. +1
    -0
      dockerbuild_gitea/notes.txt
  8. +28
    -0
      dockerbuild_goatcounter/Dockerfile
  9. +50
    -0
      dockerbuild_goatcounter/build-docker.sh
  10. +22
    -0
      dockerbuild_goatcounter/goatcounter-caddy-log-adapter/config.json
  11. +12
    -0
      dockerbuild_goatcounter/goatcounter-caddy-log-adapter/go.mod
  12. +11
    -0
      dockerbuild_goatcounter/goatcounter-caddy-log-adapter/go.sum
  13. +212
    -0
      dockerbuild_goatcounter/goatcounter-caddy-log-adapter/main.go
  14. +15
    -0
      dockerbuild_goatcounter/tail_caddy_json_log.sh
  15. +0
    -0
      dockerbuild_mariadb/.dockerignore
  16. +0
    -0
      dockerbuild_mariadb/Dockerfile
  17. +0
    -0
      dockerbuild_mariadb/build-docker.sh
  18. +0
    -0
      dockerbuild_mariadb/docker-entrypoint.sh
  19. +0
    -0
      dockerbuild_mariadb/notes.txt
  20. +3
    -0
      go.mod
  21. +0
    -0
      go.sum
  22. +12
    -2
      main.go

+ 19
- 17
build-docker.sh View File

@ -1,40 +1,42 @@
#!/bin/bash -e
VERSION="0.0.7"
VERSION="0.0.8"
rm -rf dockerbuild || true
mkdir dockerbuild
cp Dockerfile dockerbuild/Dockerfile-amd64
#cp Dockerfile dockerbuild/Dockerfile-amd64
cp Dockerfile dockerbuild/Dockerfile-arm
cp Dockerfile dockerbuild/Dockerfile-arm64
#cp Dockerfile dockerbuild/Dockerfile-arm64
sed -E 's|FROM alpine|FROM amd64/alpine|' -i dockerbuild/Dockerfile-amd64
#sed -E 's|FROM alpine|FROM amd64/alpine|' -i dockerbuild/Dockerfile-amd64
sed -E 's|FROM alpine|FROM arm32v7/alpine|' -i dockerbuild/Dockerfile-arm
sed -E 's|FROM alpine|FROM arm64v8/alpine|' -i dockerbuild/Dockerfile-arm64
#sed -E 's|FROM alpine|FROM arm64v8/alpine|' -i dockerbuild/Dockerfile-arm64
sed -E 's/GOARCH=/GOARCH=amd64/' -i dockerbuild/Dockerfile-amd64
#sed -E 's/GOARCH=/GOARCH=amd64/' -i dockerbuild/Dockerfile-amd64
sed -E 's/GOARCH=/GOARCH=arm/' -i dockerbuild/Dockerfile-arm
sed -E 's/GOARCH=/GOARCH=arm64/' -i dockerbuild/Dockerfile-arm64
#sed -E 's/GOARCH=/GOARCH=arm64/' -i dockerbuild/Dockerfile-arm64
docker build -f dockerbuild/Dockerfile-amd64 -t sequentialread/caddy-config:$VERSION-amd64 .
#docker build -f dockerbuild/Dockerfile-amd64 -t sequentialread/caddy-config:$VERSION-amd64 .
docker build -f dockerbuild/Dockerfile-arm -t sequentialread/caddy-config:$VERSION-arm .
docker build -f dockerbuild/Dockerfile-arm64 -t sequentialread/caddy-config:$VERSION-arm64 .
#docker build -f dockerbuild/Dockerfile-arm64 -t sequentialread/caddy-config:$VERSION-arm64 .
docker push sequentialread/caddy-config:$VERSION-amd64
#docker push sequentialread/caddy-config:$VERSION-amd64
docker push sequentialread/caddy-config:$VERSION-arm
docker push sequentialread/caddy-config:$VERSION-arm64
#docker push sequentialread/caddy-config:$VERSION-arm64
export DOCKER_CLI_EXPERIMENTAL=enabled
docker manifest create sequentialread/caddy-config:$VERSION \
sequentialread/caddy-config:$VERSION-amd64 \
sequentialread/caddy-config:$VERSION-arm \
sequentialread/caddy-config:$VERSION-arm64
docker manifest create sequentialread/caddy-config:$VERSION sequentialread/caddy-config:$VERSION-arm
docker manifest annotate --arch amd64 sequentialread/caddy-config:$VERSION sequentialread/caddy-config:$VERSION-amd64
# docker manifest create sequentialread/caddy-config:$VERSION \
# sequentialread/caddy-config:$VERSION-amd64 \
# sequentialread/caddy-config:$VERSION-arm \
# sequentialread/caddy-config:$VERSION-arm64
#docker manifest annotate --arch amd64 sequentialread/caddy-config:$VERSION sequentialread/caddy-config:$VERSION-amd64
docker manifest annotate --arch arm sequentialread/caddy-config:$VERSION sequentialread/caddy-config:$VERSION-arm
docker manifest annotate --arch arm64 sequentialread/caddy-config:$VERSION sequentialread/caddy-config:$VERSION-arm64
#docker manifest annotate --arch arm64 sequentialread/caddy-config:$VERSION sequentialread/caddy-config:$VERSION-arm64
docker manifest push sequentialread/caddy-config:$VERSION

+ 23
- 0
caddy/config/config.json View File

@ -6,5 +6,28 @@
"config": {
"persist": false
}
},
"logging": {
"logs": {
"default": {
"exclude": [
"http.log.access.goatcounter"
]
},
"goatcounter": {
"encoder": {
"format": "console"
},
"include": [
"http.log.access.goatcounter"
],
"writer": {
"filename": "/var/log/caddy-goatcounter.log",
"output": "file",
"roll_keep": 10,
"roll_size_mb": 1
}
}
}
}
}

+ 3
- 0
docker-compose.yml View File

@ -19,6 +19,9 @@ services:
- type: bind
source: ./caddy/data
target: /data
- type: bind
source: ./caddy/log
target: /var/log
- type: volume
source: caddy-socket-volume
target: /caddysocket


z_ghost_build/Dockerfile → dockerbuild_ghost/Dockerfile View File


z_ghost_build/docker-entrypoint.sh → dockerbuild_ghost/docker-entrypoint.sh View File


z_gitea_build/build-docker.sh → dockerbuild_gitea/build-docker.sh View File


+ 1
- 0
dockerbuild_gitea/notes.txt View File

@ -0,0 +1 @@
gitea admin create-user --name forest --password $GITEA_FOREST_PASSWORD --admin --email forest.n.johnson@gmail.com

+ 28
- 0
dockerbuild_goatcounter/Dockerfile View File

@ -0,0 +1,28 @@
FROM golang:1.16-buster as build
ARG GOARCH=
ENV CGO_ENABLED 1
ENV CC "$CC"
RUN mkdir /build
WORKDIR /build
RUN apt update && apt install -y git build-essential
RUN git clone -v https://github.com/zgoat/goatcounter
RUN cd goatcounter && git checkout be3116c320b40f1161584e78329d0ca6cb5f2dcf
RUN cd goatcounter && echo "CC=$CC" && go get
RUN cd goatcounter/cmd/goatcounter && go build -tags 'sqlite_omit_load_extension osusergo netgo' -ldflags='-extldflags=-static' -o ../../../goatcounter_binary -v .
COPY goatcounter-caddy-log-adapter /build/goatcounter-caddy-log-adapter
RUN cd /build/goatcounter-caddy-log-adapter && go get
RUN cd /build/goatcounter-caddy-log-adapter && go build -tags 'osusergo netgo' -ldflags='-extldflags=-static' -o ../../../goatcounter-caddy-log-adapter -v .
FROM alpine
WORKDIR /app
RUN apk add --update --no-cache tzdata alpine-conf jq && setup-timezone -z UTC
COPY --from=build /build/goatcounter_binary /app/goatcounter
COPY --from=build /build/goatcounter-caddy-log-adapter /app/goatcounter-caddy-log-adapter
COPY goatcounter-caddy-log-adapter/config.json /app/config.json
RUN chmod +x /app/goatcounter && chmod +x /app/goatcounter-caddy-log-adapter
ENTRYPOINT ["/app/goatcounter"]
CMD ["help"]

+ 50
- 0
dockerbuild_goatcounter/build-docker.sh View File

@ -0,0 +1,50 @@
#!/bin/bash -e
VERSION="1.4.2-15"
rm -rf dockerbuild || true
mkdir dockerbuild
#cp Dockerfile dockerbuild/Dockerfile-amd64
cp Dockerfile dockerbuild/Dockerfile-arm
#cp Dockerfile dockerbuild/Dockerfile-arm64
#sed -E 's|FROM alpine|FROM amd64/alpine|' -i dockerbuild/Dockerfile-amd64
sed -E 's|FROM alpine|FROM arm32v7/alpine|' -i dockerbuild/Dockerfile-arm
sed -E 's/GOARCH=/GOARCH=arm/' -i dockerbuild/Dockerfile-arm
sed -E 's/CC "\$CC"/CC "arm-linux-gnueabi-gcc"/' -i dockerbuild/Dockerfile-arm
sed -E 's/build-essential/gcc-arm* build-essential/' -i dockerbuild/Dockerfile-arm
#sed -E 's|FROM alpine|FROM arm64v8/alpine|' -i dockerbuild/Dockerfile-arm64
#sed -E 's/GOARCH=/GOARCH=amd64/' -i dockerbuild/Dockerfile-amd64
#aarch64-linux-musl-gcc
#cat dockerbuild/Dockerfile-arm
#sed -E 's/GOARCH=/GOARCH=arm64/' -i dockerbuild/Dockerfile-arm64
#docker build -f dockerbuild/Dockerfile-amd64 -t sequentialread/goatcounter:$VERSION-amd64 .
docker build -f dockerbuild/Dockerfile-arm -t sequentialread/goatcounter:$VERSION-arm .
#docker build -f dockerbuild/Dockerfile-arm64 -t sequentialread/goatcounter:$VERSION-arm64 .
#docker push sequentialread/goatcounter:$VERSION-amd64
docker push sequentialread/goatcounter:$VERSION-arm
#docker push sequentialread/goatcounter:$VERSION-arm64
export DOCKER_CLI_EXPERIMENTAL=enabled
docker manifest create sequentialread/goatcounter:$VERSION sequentialread/goatcounter:$VERSION-arm
# docker manifest create sequentialread/goatcounter:$VERSION \
# sequentialread/goatcounter:$VERSION-amd64 \
# sequentialread/goatcounter:$VERSION-arm \
# sequentialread/goatcounter:$VERSION-arm64
#docker manifest annotate --arch amd64 sequentialread/goatcounter:$VERSION sequentialread/goatcounter:$VERSION-amd64
docker manifest annotate --arch arm sequentialread/goatcounter:$VERSION sequentialread/goatcounter:$VERSION-arm
#docker manifest annotate --arch arm64 sequentialread/goatcounter:$VERSION sequentialread/goatcounter:$VERSION-arm64
docker manifest push sequentialread/goatcounter:$VERSION
rm -rf dockerbuild || true

+ 22
- 0
dockerbuild_goatcounter/goatcounter-caddy-log-adapter/config.json View File

@ -0,0 +1,22 @@
{
"DomainAliases": [
{
"Search": "beta.sequentialread.com",
"Replace": "b.sqr"
},
{
"Search": "sequentialread.com",
"Replace": "sqr"
}
],
"IncludeDomainInKey": true,
"IncludeSuccessOrFailureInKey": true,
"URIQuery": "drop",
"GlobalContentTypeBlacklistRegex": "(font/.*)|([^/]+/(javascript|css|less|sass))",
"Domains": [
{
"MatchHostnameRegex": "^(goatcounter|git|stream|pwm|captcha|comments\\.)?(beta\\.)?sequentialread.com",
"ContentTypeWhitelistRegex": "[^/]+/html"
}
]
}

+ 12
- 0
dockerbuild_goatcounter/goatcounter-caddy-log-adapter/go.mod View File

@ -0,0 +1,12 @@
module git.sequentialread.com/forest/sequentialread-caddy-config/dockerbuild_goatcounter/goatcounter-caddy-log-adapter
go 1.16
require (
git.sequentialread.com/forest/influx-style-env-override v0.0.0-20161210012634-dad3f15fa3dd
git.sequentialread.com/forest/pkg-errors v0.9.2 // indirect
github.com/hpcloud/tail v1.0.0
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect
gopkg.in/fsnotify.v1 v1.4.7 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
)

+ 11
- 0
dockerbuild_goatcounter/goatcounter-caddy-log-adapter/go.sum View File

@ -0,0 +1,11 @@
git.sequentialread.com/forest/influx-style-env-override v0.0.0-20161210012634-dad3f15fa3dd h1:hT6Yz1VyourG5ECUL6NT+fP/UHYDXFSa01OzWx/S5V4=
git.sequentialread.com/forest/influx-style-env-override v0.0.0-20161210012634-dad3f15fa3dd/go.mod h1:ZzXXPBYB8N8aGzrHKrRVdVDE2yZSBddmmfzs/Kfft8M=
git.sequentialread.com/forest/pkg-errors v0.9.2/go.mod h1:8TkJ/f8xLWFIAid20aoqgDZcCj9QQt+FU+rk415XO1w=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

+ 212
- 0
dockerbuild_goatcounter/goatcounter-caddy-log-adapter/main.go View File

@ -0,0 +1,212 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"reflect"
"regexp"
"strings"
envOverride "git.sequentialread.com/forest/influx-style-env-override"
"github.com/hpcloud/tail"
)
type Config struct {
DomainAliases []*DomainAlias
IncludeDomainInKey bool
IncludeMethodInKey bool
IncludeSuccessOrFailureInKey bool
URIQuery string
GlobalContentTypeBlacklistRegex string
GlobalContentTypeBlacklist *regexp.Regexp `json:"-"`
Domains []*Domain
}
type Domain struct {
MatchHostnameRegex string
ContentTypeWhitelistRegex string
URIQuery string
MatchHostname *regexp.Regexp `json:"-"`
ContentTypeWhitelist *regexp.Regexp `json:"-"`
}
type DomainAlias struct {
Search string
Replace string
}
type CaddyLog struct {
CommonLog string `json:"common_log"`
StatusCode int `json:"status"`
Request CaddyLogRequest `json:"request"`
ResponseHeaders map[string][]string `json:"resp_headers"`
}
type CaddyLogRequest struct {
URI string `json:"uri"`
Host string `json:"host"`
Method string `json:"method"`
Headers map[string][]string `json:"headers"`
}
func main() {
config := loadConfigFromFileAndEnvVars()
bytez, err := json.MarshalIndent(config, "", " ")
if err != nil {
panic(err)
}
fmt.Fprintf(os.Stderr, "goatcounter-caddy-log-adapter using config: %s", string(bytez))
logTailer, err := tail.TailFile(os.Args[0], tail.Config{ReOpen: true, Follow: true})
if err != nil {
panic(err)
}
for line := range logTailer.Lines {
if line.Err != nil {
fmt.Fprintf(os.Stderr, "unable to read next log line because %s (line: '%s')", err, line.Text)
continue
}
var caddyLog CaddyLog
err := json.Unmarshal([]byte(line.Text), &caddyLog)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to unmarshal log line because %s (line: '%s')", err, line.Text)
continue
}
referrer := ""
userAgent := ""
contentType := ""
if caddyLog.Request.Headers != nil {
referrerHeader, hasReferrerHeader := caddyLog.Request.Headers["Referrer"]
if hasReferrerHeader && len(referrerHeader) > 0 {
referrer = referrerHeader[0]
}
userAgentHeader, hasUserAgent := caddyLog.Request.Headers["User-Agent"]
if hasUserAgent && len(userAgentHeader) > 0 {
userAgent = userAgentHeader[0]
}
}
if caddyLog.ResponseHeaders != nil {
contentTypeHeader, hasContentTypeHeader := caddyLog.ResponseHeaders["Content-Type"]
if hasContentTypeHeader && len(contentTypeHeader) > 0 {
split := strings.Split(contentTypeHeader[0], ";")
contentType = strings.TrimSpace(split[0])
}
}
if config.GlobalContentTypeBlacklist != nil && config.GlobalContentTypeBlacklist.MatchString(contentType) {
continue
}
var requestDomain *Domain
for _, domain := range config.Domains {
if domain.MatchHostname.MatchString(caddyLog.Request.Host) {
requestDomain = domain
break
}
}
uriQuery := config.URIQuery
if requestDomain != nil {
if requestDomain.URIQuery != "" {
uriQuery = requestDomain.URIQuery
}
if !requestDomain.ContentTypeWhitelist.MatchString(contentType) {
continue
}
}
key := caddyLog.Request.URI
if uriQuery == "drop" {
split := strings.Split(key, "?")
key = split[0]
}
if config.IncludeMethodInKey {
key = fmt.Sprintf("/%s%s", caddyLog.Request.Method, key)
}
if config.IncludeSuccessOrFailureInKey {
isSuccess := caddyLog.StatusCode < 400
status := "success"
if !isSuccess {
status = "error"
}
key = fmt.Sprintf("/%s%s", status, key)
}
if config.IncludeDomainInKey {
host := caddyLog.Request.Host
if config.DomainAliases != nil {
for _, alias := range config.DomainAliases {
host = strings.Replace(host, alias.Search, alias.Replace, 1)
}
}
key = fmt.Sprintf("/%s%s", host, key)
}
myCommonLog := strings.Replace(caddyLog.CommonLog, caddyLog.Request.URI, key, 1)
toPrint := fmt.Sprintf("%s:%s \"%s\" \"%s\" ", caddyLog.Request.Host, myCommonLog, referrer, userAgent)
fmt.Fprintf(os.Stdout, toPrint)
fmt.Fprintf(os.Stderr, toPrint)
}
}
func loadConfigFromFileAndEnvVars() *Config {
configBytes, err := ioutil.ReadFile("config.json")
if err != nil {
panic(err)
}
config := &Config{
DomainAliases: []*DomainAlias{},
Domains: []*Domain{},
}
err = json.Unmarshal(configBytes, config)
if err != nil {
panic(err)
}
// if I remember correctly I never got the silly ApplyInfluxStyleEnvOverrides thing to be able to add new entries in a list
// so I will prepopulate the lists with 10 entries and filter the empty ones out later
for i := 0; i < 10; i++ {
config.DomainAliases = append(config.DomainAliases, &DomainAlias{})
}
for i := 0; i < 10; i++ {
config.Domains = append(config.Domains, &Domain{})
}
envOverride.ApplyInfluxStyleEnvOverrides("LOGADAPTER", reflect.ValueOf(config))
actualDomainAliases := []*DomainAlias{}
actualDomains := []*Domain{}
for _, alias := range config.DomainAliases {
if alias.Search != "" {
actualDomainAliases = append(actualDomainAliases, alias)
}
}
for _, domain := range config.Domains {
if domain.MatchHostnameRegex != "" {
if domain.ContentTypeWhitelistRegex != "" {
domain.ContentTypeWhitelist = regexp.MustCompile(domain.ContentTypeWhitelistRegex)
} else {
domain.ContentTypeWhitelist = regexp.MustCompile(".*")
}
domain.MatchHostname = regexp.MustCompile(domain.MatchHostnameRegex)
actualDomains = append(actualDomains, domain)
}
}
config.DomainAliases = actualDomainAliases
config.Domains = actualDomains
if config.GlobalContentTypeBlacklistRegex != "" {
config.GlobalContentTypeBlacklist = regexp.MustCompile(config.GlobalContentTypeBlacklistRegex)
}
return config
}

+ 15
- 0
dockerbuild_goatcounter/tail_caddy_json_log.sh View File

@ -0,0 +1,15 @@
#!/bin/sh -e
logfilepath="$1"
tail -f "$logfilepath" \
| jq -r '"\(.request.host):\(.common_log) \"\(.request.headers.Referer[0])\" \"\(.request.headers."User-Agent"[0])\""' \
| /app/goatcounter import -site https://goatcounter.beta.sequentialread.com -format combined-vhost -- -
-datetime "02/Jan/2006:15:04:05 -0700"
entrypoint: ["/bin/sh"]
command: ["-c", "./goatcounter-caddy-log-adapter /caddylog/caddy-goatcounter.log | ./goatcounter import -site https://goatcounter.beta.sequentialread.com -format combined-vhost -- -"]

z_mariadb_build/.dockerignore → dockerbuild_mariadb/.dockerignore View File


z_mariadb_build/Dockerfile → dockerbuild_mariadb/Dockerfile View File


z_mariadb_build/build-docker.sh → dockerbuild_mariadb/build-docker.sh View File


z_mariadb_build/docker-entrypoint.sh → dockerbuild_mariadb/docker-entrypoint.sh View File


z_mariadb_build/notes.txt → dockerbuild_mariadb/notes.txt View File


+ 3
- 0
go.mod View File

@ -0,0 +1,3 @@
module git.sequentialread.com/forest/sequentialread-caddy-config
go 1.16

+ 0
- 0
go.sum View File


+ 12
- 2
main.go View File

@ -85,8 +85,13 @@ type CaddyACMEIssuer struct {
}
type CaddyServer struct {
Listen []string `json:"listen"`
Routes []CaddyRoute `json:"routes"`
Listen []string `json:"listen"`
Routes []CaddyRoute `json:"routes"`
Logs *CaddyServerLogs `json:"logs"`
}
type CaddyServerLogs struct {
LoggerNames map[string]string `json:"logger_names"`
}
type CaddyRoute struct {
@ -289,6 +294,11 @@ func IngressConfig() error {
"srv0": {
Listen: []string{":443"},
Routes: []CaddyRoute{fbclidRoute},
Logs: &CaddyServerLogs{
LoggerNames: map[string]string{
"*": "goatcounter",
},
},
},
},
}


Loading…
Cancel
Save