From e70a44a146d5a87081ef1ce7a37b5df3785fdcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 1 Mar 2022 18:50:31 +0100 Subject: [PATCH] Manager: switch from PostgreSQL to SQLite This includes a modified copy of the Gorm SQLite backend, adjusted to use https://modernc.org/sqlite instead. --- README.md | 29 +- cmd/flamenco-manager/main.go | 11 - go.mod | 15 +- go.sum | 136 +----- internal/manager/config/defaults.go | 2 +- internal/manager/persistence/db.go | 7 +- .../manager/persistence/initialisation.go | 135 ------ internal/manager/persistence/jobs_test.go | 2 +- .../manager/persistence/task_scheduler.go | 4 +- internal/manager/persistence/test_support.go | 27 +- pkg/gorm-modernc-sqlite/License | 21 + pkg/gorm-modernc-sqlite/README.md | 17 + pkg/gorm-modernc-sqlite/ddlmod.go | 216 +++++++++ pkg/gorm-modernc-sqlite/ddlmod_test.go | 197 +++++++++ pkg/gorm-modernc-sqlite/errors.go | 7 + pkg/gorm-modernc-sqlite/migrator.go | 418 ++++++++++++++++++ pkg/gorm-modernc-sqlite/sqlite.go | 219 +++++++++ pkg/gorm-modernc-sqlite/sqlite_test.go | 81 ++++ 18 files changed, 1199 insertions(+), 345 deletions(-) delete mode 100644 internal/manager/persistence/initialisation.go create mode 100644 pkg/gorm-modernc-sqlite/License create mode 100644 pkg/gorm-modernc-sqlite/README.md create mode 100644 pkg/gorm-modernc-sqlite/ddlmod.go create mode 100644 pkg/gorm-modernc-sqlite/ddlmod_test.go create mode 100644 pkg/gorm-modernc-sqlite/errors.go create mode 100644 pkg/gorm-modernc-sqlite/migrator.go create mode 100644 pkg/gorm-modernc-sqlite/sqlite.go create mode 100644 pkg/gorm-modernc-sqlite/sqlite_test.go diff --git a/README.md b/README.md index b5222e19..8bad291f 100644 --- a/README.md +++ b/README.md @@ -29,31 +29,6 @@ This also gives you the Yarn package manager, which can be used to install web d Flamenco Manager has a SwaggerUI interface at http://localhost:8080/api/swagger-ui/ -## Flamenco Manager DB development machine setup. +## Database -Install PostgreSQL, then run: - -``` -sudo -u postgres createuser -D -P flamenco # give it the password 'flamenco' -sudo -u postgres createdb -O flamenco -E utf8 flamenco -sudo -u postgres createdb -O flamenco -E utf8 flamenco-test -echo "alter schema public owner to flamenco;" | sudo -u postgres psql flamenco-test -``` - -### Windows - -On Windows, add `C:\Program Files\PostgreSQL\14\bin` to your `PATH` environment variable. -Replace `14` with the version of PostgreSQL you're using. Then run: - - -``` -createuser -U postgres -D -P flamenco # give it the password 'flamenco' -createdb -U postgres -O flamenco -E utf8 flamenco -createdb -U postgres -O flamenco -E utf8 flamenco-test -psql -c "alter schema public owner to flamenco" flamenco-test postgres -``` - -When it asks "Enter password for new role:", give the password "flamenco" -When it asks "Password:", give the password for the postgres admin user (you chose this during installation of PostgreSQL). - -If you're like me, and you use Git Bash, prefix the commands with `winpty`. +Flamenco Manager includes a copy of https://github.com/go-gorm/sqlite.git, adjusted to use the pure-Go SQLite from https://modernc.org/sqlite. diff --git a/cmd/flamenco-manager/main.go b/cmd/flamenco-manager/main.go index a2f1b49f..5a1d0652 100644 --- a/cmd/flamenco-manager/main.go +++ b/cmd/flamenco-manager/main.go @@ -49,7 +49,6 @@ import ( var cliArgs struct { version bool - initDB bool } func main() { @@ -70,15 +69,6 @@ func main() { configService := config.NewService() configService.Load() - if cliArgs.initDB { - log.Info().Msg("creating databases") - err := persistence.InitialSetup() - if err != nil { - log.Fatal().Err(err).Msg("problem performing initial setup") - } - return - } - // TODO: enable TLS via Let's Encrypt. listen := configService.Get().Listen _, port, _ := net.SplitHostPort(listen) @@ -142,7 +132,6 @@ func parseCliArgs() { var quiet, debug, trace bool flag.BoolVar(&cliArgs.version, "version", false, "Shows the application version, then exits.") - flag.BoolVar(&cliArgs.initDB, "initdb", false, "Create the database; requires admin access to PostgreSQL.") flag.BoolVar(&quiet, "quiet", false, "Only log warning-level and worse.") flag.BoolVar(&debug, "debug", false, "Enable debug-level logging.") flag.BoolVar(&trace, "trace", false, "Enable trace-level logging.") diff --git a/go.mod b/go.mod index 2c720ab4..4da31062 100644 --- a/go.mod +++ b/go.mod @@ -19,10 +19,8 @@ require ( github.com/ziflex/lecho/v3 v3.1.0 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/net v0.0.0-20211013171255-e13a2654a71e - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 gopkg.in/yaml.v2 v2.4.0 - gorm.io/driver/postgres v1.0.8 - gorm.io/gorm v1.21.4 + gorm.io/gorm v1.23.2 modernc.org/sqlite v1.14.6 ) @@ -33,28 +31,17 @@ require ( github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.19.5 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.8.0 // indirect - github.com/jackc/pgio v1.0.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.0.7 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.6.2 // indirect - github.com/jackc/pgx/v4 v4.10.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.4 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/labstack/gommon v0.3.1 // indirect - github.com/lib/pq v1.10.0 // indirect github.com/mailru/easyjson v0.7.0 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect - github.com/shopspring/decimal v1.2.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect golang.org/x/mod v0.4.2 // indirect diff --git a/go.sum b/go.sum index eabe4064..8aed14d6 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,6 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -46,12 +40,8 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.7.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= @@ -64,7 +54,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -75,75 +64,19 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graarh/golang-socketio v0.0.0-20170510162725-2c44953b9b5f h1:utzdm9zUvVWGRtIpkdE4+36n+Gv60kNb7mFvgGxLElY= github.com/graarh/golang-socketio v0.0.0-20170510162725-2c44953b9b5f/go.mod h1:8gudiNCFh3ZfvInknmoXzPeV17FSH+X2J5k2cUPIwnA= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= -github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= -github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= -github.com/jackc/pgconn v1.8.0 h1:FmjZ0rOyXTr1wfWs45i4a9vjnjWUAGpMuQLD9OSs+lw= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.0.7 h1:6Pwi1b3QdY65cuv6SyVO0FgPd5J3Bl7wf/nQQjinHMA= -github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= -github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= -github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= -github.com/jackc/pgtype v1.6.2 h1:b3pDeuhbbzBYcg5kwNmNDun4pFUD/0AAr1kLXZLeNt8= -github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= -github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= -github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= -github.com/jackc/pgx/v4 v4.10.1 h1:/6Q3ye4myIj6AaplUm+eRcz4OhK9HAvFf4ePsG40LJY= -github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -162,27 +95,17 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.7/go.mod h1:bw24IXWbavc0R2RsOtpXL7RtMyP589yZ1+L7kd09ZGA= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= -github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -195,35 +118,19 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -242,24 +149,10 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziflex/lecho/v3 v3.1.0 h1:65bSzSc0yw7EEhi44lMnkOI877ZzbE7tGDWfYCQXZwI= github.com/ziflex/lecho/v3 v3.1.0/go.mod h1:dwQ6xCAKmSBHhwZ6XmiAiDptD7iklVkW7xQYGUncX0Q= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= @@ -268,16 +161,12 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -290,14 +179,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -322,8 +207,6 @@ golang.org/x/sys v0.0.0-20211103235746-7861aae1554b h1:1VkfZQv42XQlA/jchYumAnv1U golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -334,12 +217,6 @@ golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -347,8 +224,6 @@ golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -362,7 +237,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -371,14 +245,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.0.8 h1:PAgM+PaHOSAeroTjHkCHCBIHHoBIf9RgPWGo8dF2DA8= -gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= -gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.21.4 h1:J0xfPJMRfHgpVcYLrEAIqY/apdvTIkrltPQNHQLq9Qc= -gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gorm.io/gorm v1.23.1 h1:aj5IlhDzEPsoIyOPtTRVI+SyaN1u6k613sbt4pwbxG0= -gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +gorm.io/gorm v1.23.2 h1:xmq9QRMWL8HTJyhAUBXy8FqIIQCYESeKfJL4DoGKiWQ= +gorm.io/gorm v1.23.2/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= diff --git a/internal/manager/config/defaults.go b/internal/manager/config/defaults.go index e49b47bd..34d6fb38 100644 --- a/internal/manager/config/defaults.go +++ b/internal/manager/config/defaults.go @@ -34,7 +34,7 @@ var defaultConfig = Conf{ ManagerName: "Flamenco Manager", Listen: ":8080", ListenHTTPS: ":8433", - DatabaseDSN: "host=localhost user=flamenco password=flamenco dbname=flamenco TimeZone=Europe/Amsterdam", + DatabaseDSN: "flamenco-manager.sqlite", TaskLogsPath: "./task-logs", // DownloadTaskSleep: 10 * time.Minute, // DownloadTaskRecheckThrottle: 10 * time.Second, diff --git a/internal/manager/persistence/db.go b/internal/manager/persistence/db.go index 62bcedf0..1d784875 100644 --- a/internal/manager/persistence/db.go +++ b/internal/manager/persistence/db.go @@ -25,8 +25,9 @@ import ( "context" "github.com/rs/zerolog/log" - "gorm.io/driver/postgres" "gorm.io/gorm" + + sqlite "gitlab.com/blender/flamenco-ng-poc/pkg/gorm-modernc-sqlite" ) // DB provides the database interface. @@ -51,7 +52,9 @@ func openDB(ctx context.Context, uri string) (*DB, error) { // TODO: don't log the password. log.Info().Str("dsn", uri).Msg("opening database") - gormDB, err := gorm.Open(postgres.Open(uri), &gorm.Config{}) + connection := sqlite.Open(uri) + // connection := postgres.Open(uri) + gormDB, err := gorm.Open(connection, &gorm.Config{}) if err != nil { return nil, err } diff --git a/internal/manager/persistence/initialisation.go b/internal/manager/persistence/initialisation.go deleted file mode 100644 index ca7e0c45..00000000 --- a/internal/manager/persistence/initialisation.go +++ /dev/null @@ -1,135 +0,0 @@ -package persistence - -/* ***** BEGIN GPL LICENSE BLOCK ***** - * - * Original Code Copyright (C) 2022 Blender Foundation. - * - * This file is part of Flamenco. - * - * Flamenco is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - * A PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along with - * Flamenco. If not, see . - * - * ***** END GPL LICENSE BLOCK ***** */ - -import ( - "bufio" - "database/sql" - "errors" - "fmt" - "os" - "runtime" - - "github.com/rs/zerolog/log" - "golang.org/x/term" - "gorm.io/driver/postgres" - "gorm.io/gorm" -) - -var errInputTooLong = errors.New("input is too long") - -const adminDSN = "host=localhost user=postgres password=%s dbname=%s TimeZone=Europe/Amsterdam" - -// InitialSetup uses the `postgres` admin user to set up the database. -// TODO: distinguish between production and development setups. -func InitialSetup() error { - // Get the password of the 'postgres' user. - adminPass, err := readPassword() - if err != nil { - return fmt.Errorf("unable to read password: %w", err) - } - - // Connect to the 'postgres' database so we can create other databases. - db, err := connectDBAsAdmin(adminPass, "postgres") - if err != nil { - return fmt.Errorf("unable to connect to the database: %w", err) - } - - // TODO: get username / password / database name from some config file, user input, CLI args, whatevah. - // Has to be used by the regular Flamenco Manager runs as well, though. - username := "flamenco" - userPass := "flamenco" - - tx := db.Exec("CREATE USER @user PASSWORD @pass NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT LOGIN", - sql.Named("user", username), - sql.Named("pass", userPass)) - if tx.Error != nil { - return fmt.Errorf("unable to create database user '%s': %w", username, tx.Error) - } - - // Create the databases. - tx = db.Debug().Exec("CREATE DATABASE flamenco OWNER ? ENCODING 'utf8'", username) - if tx.Error != nil { - return fmt.Errorf("unable to create database 'flamenco': %w", tx.Error) - } - tx = db.Exec("CREATE DATABASE flamenco-test OWNER ? ENCODING 'utf8'", username) - if tx.Error != nil { - return fmt.Errorf("unable to create database 'flamenco': %w", tx.Error) - } - - // Close the connection so we can reconnect. - sqlDB, err := db.DB() - if err != nil { - fmt.Printf("error closing the database connection, please report this issue: %v", err) - } else { - sqlDB.Close() - } - - // Allow 'flamenco' user to completely nuke and recreate the flamenco-test database, without needing 'CREATEDB' permission. - db, err = connectDBAsAdmin(adminPass, "flamenco-test") - if err != nil { - return fmt.Errorf("unable to reconnect to the database: %w", err) - } - tx = db.Exec("ALTER SCHEMA public OWNER TO ?", username) - if tx.Error != nil { - fmt.Printf("Unable to allow database user '%s' to reset the test database: %v\n", username, tx.Error) - fmt.Println("This is not an issue, unless you want to develop Flamenco yourself.") - } - - return nil -} - -func readPassword() (string, error) { - if pwFromEnv := os.Getenv("PSQL_ADMIN"); pwFromEnv != "" { - log.Info().Msg("getting password from PSQL_ADMIN environment variable") - return pwFromEnv, nil - } - - fmt.Print("PostgreSQL admin password: ") - - var ( - line []byte - err error - ) - - if runtime.GOOS == "windows" { - // term.ReadPassword() doesn't work reliably on Windows, especially when you - // use a MingW terminal (like Git Bash). See - // https://github.com/golang/go/issues/11914#issuecomment-613715787 for more - // info. - // - // The downside is that this echoes the password to the terminal. - buf := bufio.NewReader(os.Stdin) - line, _, err = buf.ReadLine() - } else { - fd := int(os.Stdin.Fd()) - line, err = term.ReadPassword(fd) - } - if err != nil { - return "", err - } - return string(line), nil -} - -func connectDBAsAdmin(password, database string) (*gorm.DB, error) { - dsn := fmt.Sprintf(adminDSN, password, database) - return gorm.Open(postgres.Open(dsn), &gorm.Config{}) -} diff --git a/internal/manager/persistence/jobs_test.go b/internal/manager/persistence/jobs_test.go index 83e77970..6deb31f0 100644 --- a/internal/manager/persistence/jobs_test.go +++ b/internal/manager/persistence/jobs_test.go @@ -274,7 +274,7 @@ func createTestAuthoredJobWithTasks() job_compilers.AuthoredJob { Priority: 50, Settings: job_compilers.JobSettings{ "frames": "1-6", - "chunk_size": 3.0, // The roundtrip to JSON in PostgreSQL can make this a float. + "chunk_size": 3.0, // The roundtrip to JSON in the database can make this a float. }, Metadata: job_compilers.JobMetadata{ "author": "Sybren", diff --git a/internal/manager/persistence/task_scheduler.go b/internal/manager/persistence/task_scheduler.go index f063fa13..e77ea9f5 100644 --- a/internal/manager/persistence/task_scheduler.go +++ b/internal/manager/persistence/task_scheduler.go @@ -100,8 +100,8 @@ func findTaskForWorker(tx *gorm.DB, w *Worker) (*Task, error) { Where("tasks.type in ?", w.TaskTypes()). // Supported task types Where("tasks.worker_id = ? or tasks.worker_id is NULL", w.ID). // assigned to this worker or not assigned at all // TODO: Non-blacklisted - Order("jobs.priority desc"). // Highest job priority - Order("priority desc"). // Highest task priority + Order("jobs.priority desc"). // Highest job priority + Order("tasks.priority desc"). // Highest task priority Limit(1). Preload("Job"). Find(&task) diff --git a/internal/manager/persistence/test_support.go b/internal/manager/persistence/test_support.go index afbd7e9e..1089c32a 100644 --- a/internal/manager/persistence/test_support.go +++ b/internal/manager/persistence/test_support.go @@ -22,40 +22,31 @@ package persistence * ***** END GPL LICENSE BLOCK ***** */ import ( + "os" "testing" "time" "github.com/stretchr/testify/assert" "golang.org/x/net/context" - "gorm.io/gorm" ) -const TestDSN = "host=localhost user=flamenco password=flamenco dbname=flamenco-test TimeZone=Europe/Amsterdam" +const TestDSN = "flamenco-test.sqlite" func CreateTestDB(t *testing.T) *DB { // Creating a new database should be fast. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() + if _, err := os.Stat(TestDSN); err == nil { + // File exists. + if err := os.Remove(TestDSN); err != nil { + t.Fatalf("unable to remove %s: %v", TestDSN, err) + } + } + db, err := openDB(ctx, TestDSN) assert.NoError(t, err) - // Erase everything in the database. - var tx *gorm.DB - tx = db.gormDB.Exec("DROP SCHEMA IF EXISTS public CASCADE") - if tx.Error != nil { - t.Fatalf("error dropping database schema: %v", tx.Error) - } - assert.NoError(t, tx.Error) - tx = db.gormDB.Exec("CREATE SCHEMA public") - assert.NoError(t, tx.Error) - - // Restore default grants (source: https://stackoverflow.com/questions/3327312/how-can-i-drop-all-the-tables-in-a-postgresql-database) - tx = db.gormDB.Exec("GRANT ALL ON SCHEMA public TO postgres") - assert.NoError(t, tx.Error) - tx = db.gormDB.Exec("GRANT ALL ON SCHEMA public TO public") - assert.NoError(t, tx.Error) - err = db.migrate() assert.NoError(t, err) diff --git a/pkg/gorm-modernc-sqlite/License b/pkg/gorm-modernc-sqlite/License new file mode 100644 index 00000000..037e1653 --- /dev/null +++ b/pkg/gorm-modernc-sqlite/License @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013-NOW Jinzhu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pkg/gorm-modernc-sqlite/README.md b/pkg/gorm-modernc-sqlite/README.md new file mode 100644 index 00000000..91876ae8 --- /dev/null +++ b/pkg/gorm-modernc-sqlite/README.md @@ -0,0 +1,17 @@ +# GORM Sqlite Driver + +![CI](https://github.com/go-gorm/sqlite/workflows/CI/badge.svg) + +## USAGE + +```go +import ( + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// modernc.org/sqlite +db, err := gorm.Open(sqlite.Open("gorm.db"), &gorm.Config{}) +``` + +Checkout [https://gorm.io](https://gorm.io) for details. diff --git a/pkg/gorm-modernc-sqlite/ddlmod.go b/pkg/gorm-modernc-sqlite/ddlmod.go new file mode 100644 index 00000000..73c20ea0 --- /dev/null +++ b/pkg/gorm-modernc-sqlite/ddlmod.go @@ -0,0 +1,216 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "regexp" + "strings" + + "gorm.io/gorm/migrator" +) + +var ( + sqliteSeparator = "`|\"|'|\t" + indexRegexp = regexp.MustCompile(fmt.Sprintf("CREATE(?: UNIQUE)? INDEX [%v][\\w\\d]+[%v] ON (.*)$", sqliteSeparator, sqliteSeparator)) + tableRegexp = regexp.MustCompile(fmt.Sprintf("(?i)(CREATE TABLE [%v]?[\\w\\d]+[%v]?)(?: \\((.*)\\))?", sqliteSeparator, sqliteSeparator)) + separatorRegexp = regexp.MustCompile(fmt.Sprintf("[%v]", sqliteSeparator)) + columnsRegexp = regexp.MustCompile(fmt.Sprintf("\\([%v]?([\\w\\d]+)[%v]?(?:,[%v]?([\\w\\d]+)[%v]){0,}\\)", sqliteSeparator, sqliteSeparator, sqliteSeparator, sqliteSeparator)) + columnRegexp = regexp.MustCompile(fmt.Sprintf("^[%v]?([\\w\\d]+)[%v]?\\s+([\\w\\(\\)\\d]+)(.*)$", sqliteSeparator, sqliteSeparator)) + defaultValueRegexp = regexp.MustCompile("(?i) DEFAULT \\(?(.+)?\\)?( |COLLATE|GENERATED|$)") +) + +type ddl struct { + head string + fields []string + columns []migrator.ColumnType +} + +func parseDDL(strs ...string) (*ddl, error) { + var result ddl + for _, str := range strs { + if sections := tableRegexp.FindStringSubmatch(str); len(sections) > 0 { + var ( + ddlBody = sections[2] + bracketLevel int + quote rune + buf string + ) + + result.head = sections[1] + + for idx, c := range []rune(ddlBody) { + var next rune = 0 + if idx+1 < len(ddlBody) { + next = []rune(ddlBody)[idx+1] + } + + if sc := string(c); separatorRegexp.MatchString(sc) { + if c == next { + buf += sc // Skip escaped quote + idx++ + } else if quote > 0 { + quote = 0 + } else { + quote = c + } + } else if quote == 0 { + if c == '(' { + bracketLevel++ + } else if c == ')' { + bracketLevel-- + } else if bracketLevel == 0 { + if c == ',' { + result.fields = append(result.fields, strings.TrimSpace(buf)) + buf = "" + continue + } + } + } + + if bracketLevel < 0 { + return nil, errors.New("invalid DDL, unbalanced brackets") + } + + buf += string(c) + } + + if bracketLevel != 0 { + return nil, errors.New("invalid DDL, unbalanced brackets") + } + + if buf != "" { + result.fields = append(result.fields, strings.TrimSpace(buf)) + } + + for _, f := range result.fields { + fUpper := strings.ToUpper(f) + if strings.HasPrefix(fUpper, "CHECK") || + strings.HasPrefix(fUpper, "CONSTRAINT") { + continue + } + + if strings.HasPrefix(fUpper, "PRIMARY KEY") { + matches := columnsRegexp.FindStringSubmatch(f) + if len(matches) > 1 { + for _, name := range matches[1:] { + for idx, column := range result.columns { + if column.NameValue.String == name { + column.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} + result.columns[idx] = column + break + } + } + } + } + } else if matches := columnRegexp.FindStringSubmatch(f); len(matches) > 0 { + columnType := migrator.ColumnType{ + NameValue: sql.NullString{String: matches[1], Valid: true}, + DataTypeValue: sql.NullString{String: matches[2], Valid: true}, + ColumnTypeValue: sql.NullString{String: matches[2], Valid: true}, + PrimaryKeyValue: sql.NullBool{Valid: true}, + UniqueValue: sql.NullBool{Valid: true}, + NullableValue: sql.NullBool{Valid: true}, + DefaultValueValue: sql.NullString{Valid: true}, + } + + matchUpper := strings.ToUpper(matches[3]) + if strings.Contains(matchUpper, " NOT NULL") { + columnType.NullableValue = sql.NullBool{Bool: false, Valid: true} + } else if strings.Contains(matchUpper, " NULL") { + columnType.NullableValue = sql.NullBool{Bool: true, Valid: true} + } + if strings.Contains(matchUpper, " UNIQUE") { + columnType.UniqueValue = sql.NullBool{Bool: true, Valid: true} + } + if strings.Contains(matchUpper, " PRIMARY") { + columnType.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true} + } + if defaultMatches := defaultValueRegexp.FindStringSubmatch(matches[3]); len(defaultMatches) > 1 { + columnType.DefaultValueValue = sql.NullString{String: strings.Trim(defaultMatches[1], `"`), Valid: true} + } + + result.columns = append(result.columns, columnType) + } + } + } else if matches := indexRegexp.FindStringSubmatch(str); len(matches) > 0 { + if columns := columnsRegexp.FindStringSubmatch(matches[1]); len(columns) == 1 { + for idx, c := range result.columns { + if c.NameValue.String == columns[0] { + c.UniqueValue = sql.NullBool{Bool: true, Valid: true} + result.columns[idx] = c + } + } + } + } else { + return nil, errors.New("invalid DDL") + } + } + + return &result, nil +} + +func (d *ddl) compile() string { + if len(d.fields) == 0 { + return d.head + } + + return fmt.Sprintf("%s (%s)", d.head, strings.Join(d.fields, ",")) +} + +func (d *ddl) addConstraint(name string, sql string) { + reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + + for i := 0; i < len(d.fields); i++ { + if reg.MatchString(d.fields[i]) { + d.fields[i] = sql + return + } + } + + d.fields = append(d.fields, sql) +} + +func (d *ddl) removeConstraint(name string) bool { + reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + + for i := 0; i < len(d.fields); i++ { + if reg.MatchString(d.fields[i]) { + d.fields = append(d.fields[:i], d.fields[i+1:]...) + return true + } + } + return false +} + +func (d *ddl) hasConstraint(name string) bool { + reg := regexp.MustCompile("^CONSTRAINT [\"`]?" + regexp.QuoteMeta(name) + "[\"` ]") + + for _, f := range d.fields { + if reg.MatchString(f) { + return true + } + } + return false +} + +func (d *ddl) getColumns() []string { + res := []string{} + + for _, f := range d.fields { + fUpper := strings.ToUpper(f) + if strings.HasPrefix(fUpper, "PRIMARY KEY") || + strings.HasPrefix(fUpper, "CHECK") || + strings.HasPrefix(fUpper, "CONSTRAINT") { + continue + } + + reg := regexp.MustCompile("^[\"`']?([\\w\\d]+)[\"`']?") + match := reg.FindStringSubmatch(f) + + if match != nil { + res = append(res, "`"+match[1]+"`") + } + } + return res +} diff --git a/pkg/gorm-modernc-sqlite/ddlmod_test.go b/pkg/gorm-modernc-sqlite/ddlmod_test.go new file mode 100644 index 00000000..4d1f5c7f --- /dev/null +++ b/pkg/gorm-modernc-sqlite/ddlmod_test.go @@ -0,0 +1,197 @@ +package sqlite + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "gorm.io/gorm/migrator" +) + +func TestParseDDL(t *testing.T) { + params := []struct { + name string + sql []string + nFields int + columns []migrator.ColumnType + }{ + {"with_fk", []string{ + "CREATE TABLE `notes` (`id` integer NOT NULL,`text` varchar(500) DEFAULT \"hello\",`age` integer DEFAULT 18,`user_id` integer,PRIMARY KEY (`id`),CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))", + "CREATE UNIQUE INDEX `idx_profiles_refer` ON `profiles`(`text`)", + }, 6, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "id", Valid: true}, DataTypeValue: sql.NullString{String: "integer", Valid: true}, ColumnTypeValue: sql.NullString{String: "integer", Valid: true}, PrimaryKeyValue: sql.NullBool{Bool: true, Valid: true}, NullableValue: sql.NullBool{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: true}}, + {NameValue: sql.NullString{String: "text", Valid: true}, DataTypeValue: sql.NullString{String: "varchar(500)", Valid: true}, ColumnTypeValue: sql.NullString{String: "varchar(500)", Valid: true}, DefaultValueValue: sql.NullString{String: "hello", Valid: true}, NullableValue: sql.NullBool{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "age", Valid: true}, DataTypeValue: sql.NullString{String: "integer", Valid: true}, ColumnTypeValue: sql.NullString{String: "integer", Valid: true}, DefaultValueValue: sql.NullString{String: "18", Valid: true}, NullableValue: sql.NullBool{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "user_id", Valid: true}, DataTypeValue: sql.NullString{String: "integer", Valid: true}, ColumnTypeValue: sql.NullString{String: "integer", Valid: true}, DefaultValueValue: sql.NullString{Valid: true}, NullableValue: sql.NullBool{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }, + }, + {"with_check", []string{"CREATE TABLE Persons (ID int NOT NULL,LastName varchar(255) NOT NULL,FirstName varchar(255),Age int,CHECK (Age>=18),CHECK (FirstName<>'John'))"}, 6, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "ID", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, NullableValue: sql.NullBool{Valid: true}, DefaultValueValue: sql.NullString{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "LastName", Valid: true}, DataTypeValue: sql.NullString{String: "varchar(255)", Valid: true}, ColumnTypeValue: sql.NullString{String: "varchar(255)", Valid: true}, NullableValue: sql.NullBool{Bool: false, Valid: true}, DefaultValueValue: sql.NullString{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "FirstName", Valid: true}, DataTypeValue: sql.NullString{String: "varchar(255)", Valid: true}, ColumnTypeValue: sql.NullString{String: "varchar(255)", Valid: true}, DefaultValueValue: sql.NullString{Valid: true}, NullableValue: sql.NullBool{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + {NameValue: sql.NullString{String: "Age", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, DefaultValueValue: sql.NullString{Valid: true}, NullableValue: sql.NullBool{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }}, + {"lowercase", []string{"create table test (ID int NOT NULL)"}, 1, []migrator.ColumnType{ + {NameValue: sql.NullString{String: "ID", Valid: true}, DataTypeValue: sql.NullString{String: "int", Valid: true}, ColumnTypeValue: sql.NullString{String: "int", Valid: true}, NullableValue: sql.NullBool{Bool: false, Valid: true}, DefaultValueValue: sql.NullString{Valid: true}, UniqueValue: sql.NullBool{Valid: true}, PrimaryKeyValue: sql.NullBool{Valid: true}}, + }, + }, + {"no brackets", []string{"create table test"}, 0, nil}, + } + + for _, p := range params { + t.Run(p.name, func(t *testing.T) { + ddl, err := parseDDL(p.sql...) + + if err != nil { + panic(err.Error()) + } + + assert.Equal(t, p.sql[0], ddl.compile()) + assert.Len(t, ddl.fields, p.nFields) + assert.Equal(t, ddl.columns, p.columns) + }) + } +} + +func TestParseDDL_error(t *testing.T) { + params := []struct { + name string + sql string + }{ + {"invalid_cmd", "CREATE TABLE"}, + {"unbalanced_brackets", "CREATE TABLE test (ID int NOT NULL,Name varchar(255)"}, + {"unbalanced_brackets2", "CREATE TABLE test (ID int NOT NULL,Name varchar(255)))"}, + } + + for _, p := range params { + t.Run(p.name, func(t *testing.T) { + _, err := parseDDL(p.sql) + if err == nil { + t.Fail() + } + }) + } +} + +func TestAddConstraint(t *testing.T) { + params := []struct { + name string + fields []string + cName string + sql string + expect []string + }{ + { + name: "add_new", + fields: []string{"`id` integer NOT NULL"}, + cName: "fk_users_notes", + sql: "CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))", + expect: []string{"`id` integer NOT NULL", "CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + }, + { + name: "update", + fields: []string{"`id` integer NOT NULL", "CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + cName: "fk_users_notes", + sql: "CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`)) ON UPDATE CASCADE ON DELETE CASCADE", + expect: []string{"`id` integer NOT NULL", "CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`)) ON UPDATE CASCADE ON DELETE CASCADE"}, + }, + { + name: "add_check", + fields: []string{"`id` integer NOT NULL"}, + cName: "name_checker", + sql: "CONSTRAINT `name_checker` CHECK (`name` <> 'jinzhu')", + expect: []string{"`id` integer NOT NULL", "CONSTRAINT `name_checker` CHECK (`name` <> 'jinzhu')"}, + }, + { + name: "update_check", + fields: []string{"`id` integer NOT NULL", "CONSTRAINT `name_checker` CHECK (`name` <> 'thetadev')"}, + cName: "name_checker", + sql: "CONSTRAINT `name_checker` CHECK (`name` <> 'jinzhu')", + expect: []string{"`id` integer NOT NULL", "CONSTRAINT `name_checker` CHECK (`name` <> 'jinzhu')"}, + }, + } + + for _, p := range params { + t.Run(p.name, func(t *testing.T) { + testDDL := ddl{fields: p.fields} + + testDDL.addConstraint(p.cName, p.sql) + assert.Equal(t, p.expect, testDDL.fields) + }) + } +} + +func TestRemoveConstraint(t *testing.T) { + params := []struct { + name string + fields []string + cName string + success bool + expect []string + }{ + { + name: "fk", + fields: []string{"`id` integer NOT NULL", "CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))"}, + cName: "fk_users_notes", + success: true, + expect: []string{"`id` integer NOT NULL"}, + }, + { + name: "check", + fields: []string{"CONSTRAINT `name_checker` CHECK (`name` <> 'thetadev')", "`id` integer NOT NULL"}, + cName: "name_checker", + success: true, + expect: []string{"`id` integer NOT NULL"}, + }, + { + name: "none", + fields: []string{"CONSTRAINT `name_checker` CHECK (`name` <> 'thetadev')", "`id` integer NOT NULL"}, + cName: "nothing", + success: false, + expect: []string{"CONSTRAINT `name_checker` CHECK (`name` <> 'thetadev')", "`id` integer NOT NULL"}, + }, + } + + for _, p := range params { + t.Run(p.name, func(t *testing.T) { + testDDL := ddl{fields: p.fields} + + success := testDDL.removeConstraint(p.cName) + + assert.Equal(t, p.success, success) + assert.Equal(t, p.expect, testDDL.fields) + }) + } +} + +func TestGetColumns(t *testing.T) { + params := []struct { + name string + ddl string + columns []string + }{ + { + name: "with_fk", + ddl: "CREATE TABLE `notes` (`id` integer NOT NULL,`text` varchar(500),`user_id` integer,PRIMARY KEY (`id`),CONSTRAINT `fk_users_notes` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`))", + columns: []string{"`id`", "`text`", "`user_id`"}, + }, + { + name: "with_check", + ddl: "CREATE TABLE Persons (ID int NOT NULL,LastName varchar(255) NOT NULL,FirstName varchar(255),Age int,CHECK (Age>=18),CHECK (FirstName!='John'))", + columns: []string{"`ID`", "`LastName`", "`FirstName`", "`Age`"}, + }, + } + + for _, p := range params { + t.Run(p.name, func(t *testing.T) { + testDDL, err := parseDDL(p.ddl) + if err != nil { + panic(err.Error()) + } + + cols := testDDL.getColumns() + + assert.Equal(t, p.columns, cols) + }) + } +} diff --git a/pkg/gorm-modernc-sqlite/errors.go b/pkg/gorm-modernc-sqlite/errors.go new file mode 100644 index 00000000..67b4d98a --- /dev/null +++ b/pkg/gorm-modernc-sqlite/errors.go @@ -0,0 +1,7 @@ +package sqlite + +import "errors" + +var ( + ErrConstraintsNotImplemented = errors.New("constraints not implemented on sqlite, consider using DisableForeignKeyConstraintWhenMigrating, more details https://github.com/go-gorm/gorm/wiki/GORM-V2-Release-Note-Draft#all-new-migrator") +) diff --git a/pkg/gorm-modernc-sqlite/migrator.go b/pkg/gorm-modernc-sqlite/migrator.go new file mode 100644 index 00000000..641b1f46 --- /dev/null +++ b/pkg/gorm-modernc-sqlite/migrator.go @@ -0,0 +1,418 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "regexp" + "strings" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/migrator" + "gorm.io/gorm/schema" +) + +type Migrator struct { + migrator.Migrator +} + +func (m *Migrator) RunWithoutForeignKey(fc func() error) error { + var enabled int + m.DB.Raw("PRAGMA foreign_keys").Scan(&enabled) + if enabled == 1 { + m.DB.Exec("PRAGMA foreign_keys = OFF") + defer m.DB.Exec("PRAGMA foreign_keys = ON") + } + + return fc() +} + +func (m Migrator) HasTable(value interface{}) bool { + var count int + m.Migrator.RunWithValue(value, func(stmt *gorm.Statement) error { + return m.DB.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", stmt.Table).Row().Scan(&count) + }) + return count > 0 +} + +func (m Migrator) DropTable(values ...interface{}) error { + return m.RunWithoutForeignKey(func() error { + values = m.ReorderModels(values, false) + tx := m.DB.Session(&gorm.Session{}) + + for i := len(values) - 1; i >= 0; i-- { + if err := m.RunWithValue(values[i], func(stmt *gorm.Statement) error { + return tx.Exec("DROP TABLE IF EXISTS ?", clause.Table{Name: stmt.Table}).Error + }); err != nil { + return err + } + } + + return nil + }) +} + +func (m Migrator) GetTables() (tableList []string, err error) { + return tableList, m.DB.Raw("SELECT name FROM sqlite_master where type=?", "table").Scan(&tableList).Error +} + +func (m Migrator) HasColumn(value interface{}, name string) bool { + var count int + m.Migrator.RunWithValue(value, func(stmt *gorm.Statement) error { + if stmt.Schema != nil { + if field := stmt.Schema.LookUpField(name); field != nil { + name = field.DBName + } + } + + if name != "" { + m.DB.Raw( + "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND (sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ?)", + "table", stmt.Table, `%"`+name+`" %`, `%`+name+` %`, "%`"+name+"`%", "%["+name+"]%", "%\t"+name+"\t%", + ).Row().Scan(&count) + } + return nil + }) + return count > 0 +} + +func (m Migrator) AlterColumn(value interface{}, name string) error { + return m.RunWithoutForeignKey(func() error { + return m.recreateTable(value, nil, func(rawDDL string, stmt *gorm.Statement) (sql string, sqlArgs []interface{}, err error) { + if field := stmt.Schema.LookUpField(name); field != nil { + reg, err := regexp.Compile("(`|'|\"| )" + field.DBName + "(`|'|\"| ) .*?,") + if err != nil { + return "", nil, err + } + + createSQL := reg.ReplaceAllString(rawDDL, fmt.Sprintf("`%v` ?,", field.DBName)) + + return createSQL, []interface{}{m.FullDataTypeOf(field)}, nil + } + return "", nil, fmt.Errorf("failed to alter field with name %v", name) + }) + }) +} + +// ColumnTypes return columnTypes []gorm.ColumnType and execErr error +func (m Migrator) ColumnTypes(value interface{}) ([]gorm.ColumnType, error) { + columnTypes := make([]gorm.ColumnType, 0) + execErr := m.RunWithValue(value, func(stmt *gorm.Statement) (err error) { + var ( + sqls []string + sqlDDL *ddl + ) + + if err := m.DB.Raw("SELECT sql FROM sqlite_master WHERE type IN ? AND tbl_name = ? AND sql IS NOT NULL order by type = ? desc", []string{"table", "index"}, stmt.Table, "table").Scan(&sqls).Error; err != nil { + return err + } + + if sqlDDL, err = parseDDL(sqls...); err != nil { + return err + } + + rows, err := m.DB.Session(&gorm.Session{}).Table(stmt.Table).Limit(1).Rows() + if err != nil { + return err + } + defer func() { + err = rows.Close() + }() + + var rawColumnTypes []*sql.ColumnType + rawColumnTypes, err = rows.ColumnTypes() + if err != nil { + return err + } + + for _, c := range rawColumnTypes { + columnType := migrator.ColumnType{SQLColumnType: c} + for _, column := range sqlDDL.columns { + if column.NameValue.String == c.Name() { + column.SQLColumnType = c + columnType = column + break + } + } + columnTypes = append(columnTypes, columnType) + } + + return err + }) + + return columnTypes, execErr +} + +func (m Migrator) DropColumn(value interface{}, name string) error { + return m.recreateTable(value, nil, func(rawDDL string, stmt *gorm.Statement) (sql string, sqlArgs []interface{}, err error) { + if field := stmt.Schema.LookUpField(name); field != nil { + name = field.DBName + } + + reg, err := regexp.Compile("(`|'|\"| |\\[)" + name + "(`|'|\"| |\\]) .*?,") + if err != nil { + return "", nil, err + } + + createSQL := reg.ReplaceAllString(rawDDL, "") + + return createSQL, nil, nil + }) +} + +func (m Migrator) CreateConstraint(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + constraint, chk, table := m.GuessConstraintAndTable(stmt, name) + + return m.recreateTable(value, &table, + func(rawDDL string, stmt *gorm.Statement) (sql string, sqlArgs []interface{}, err error) { + var ( + constraintName string + constraintSql string + constraintValues []interface{} + ) + + if constraint != nil { + constraintName = constraint.Name + constraintSql, constraintValues = buildConstraint(constraint) + } else if chk != nil { + constraintName = chk.Name + constraintSql = "CONSTRAINT ? CHECK (?)" + constraintValues = []interface{}{clause.Column{Name: chk.Name}, clause.Expr{SQL: chk.Constraint}} + } else { + return "", nil, nil + } + + createDDL, err := parseDDL(rawDDL) + if err != nil { + return "", nil, err + } + createDDL.addConstraint(constraintName, constraintSql) + createSQL := createDDL.compile() + + return createSQL, constraintValues, nil + }) + }) +} + +func (m Migrator) DropConstraint(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + constraint, chk, table := m.GuessConstraintAndTable(stmt, name) + if constraint != nil { + name = constraint.Name + } else if chk != nil { + name = chk.Name + } + + return m.recreateTable(value, &table, + func(rawDDL string, stmt *gorm.Statement) (sql string, sqlArgs []interface{}, err error) { + createDDL, err := parseDDL(rawDDL) + if err != nil { + return "", nil, err + } + createDDL.removeConstraint(name) + createSQL := createDDL.compile() + + return createSQL, nil, nil + }) + }) +} + +func (m Migrator) HasConstraint(value interface{}, name string) bool { + var count int64 + m.RunWithValue(value, func(stmt *gorm.Statement) error { + constraint, chk, table := m.GuessConstraintAndTable(stmt, name) + if constraint != nil { + name = constraint.Name + } else if chk != nil { + name = chk.Name + } + + m.DB.Raw( + "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND (sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ? OR sql LIKE ?)", + "table", table, `%CONSTRAINT "`+name+`" %`, `%CONSTRAINT `+name+` %`, "%CONSTRAINT `"+name+"`%", "%CONSTRAINT ["+name+"]%", "%CONSTRAINT \t"+name+"\t%", + ).Row().Scan(&count) + + return nil + }) + + return count > 0 +} + +func (m Migrator) CurrentDatabase() (name string) { + var null interface{} + m.DB.Raw("PRAGMA database_list").Row().Scan(&null, &name, &null) + return +} + +func (m Migrator) BuildIndexOptions(opts []schema.IndexOption, stmt *gorm.Statement) (results []interface{}) { + for _, opt := range opts { + str := stmt.Quote(opt.DBName) + if opt.Expression != "" { + str = opt.Expression + } + + if opt.Collate != "" { + str += " COLLATE " + opt.Collate + } + + if opt.Sort != "" { + str += " " + opt.Sort + } + results = append(results, clause.Expr{SQL: str}) + } + return +} + +func (m Migrator) CreateIndex(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + if idx := stmt.Schema.LookIndex(name); idx != nil { + opts := m.BuildIndexOptions(idx.Fields, stmt) + values := []interface{}{clause.Column{Name: idx.Name}, clause.Table{Name: stmt.Table}, opts} + + createIndexSQL := "CREATE " + if idx.Class != "" { + createIndexSQL += idx.Class + " " + } + createIndexSQL += "INDEX ?" + + if idx.Type != "" { + createIndexSQL += " USING " + idx.Type + } + createIndexSQL += " ON ??" + + if idx.Where != "" { + createIndexSQL += " WHERE " + idx.Where + } + + return m.DB.Exec(createIndexSQL, values...).Error + } + + return fmt.Errorf("failed to create index with name %v", name) + }) +} + +func (m Migrator) HasIndex(value interface{}, name string) bool { + var count int + m.RunWithValue(value, func(stmt *gorm.Statement) error { + if idx := stmt.Schema.LookIndex(name); idx != nil { + name = idx.Name + } + + if name != "" { + m.DB.Raw( + "SELECT count(*) FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "index", stmt.Table, name, + ).Row().Scan(&count) + } + return nil + }) + return count > 0 +} + +func (m Migrator) RenameIndex(value interface{}, oldName, newName string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + var sql string + m.DB.Raw("SELECT sql FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "index", stmt.Table, oldName).Row().Scan(&sql) + if sql != "" { + return m.DB.Exec(strings.Replace(sql, oldName, newName, 1)).Error + } + return fmt.Errorf("failed to find index with name %v", oldName) + }) +} + +func (m Migrator) DropIndex(value interface{}, name string) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + if idx := stmt.Schema.LookIndex(name); idx != nil { + name = idx.Name + } + + return m.DB.Exec("DROP INDEX ?", clause.Column{Name: name}).Error + }) +} + +func buildConstraint(constraint *schema.Constraint) (sql string, results []interface{}) { + sql = "CONSTRAINT ? FOREIGN KEY ? REFERENCES ??" + if constraint.OnDelete != "" { + sql += " ON DELETE " + constraint.OnDelete + } + + if constraint.OnUpdate != "" { + sql += " ON UPDATE " + constraint.OnUpdate + } + + var foreignKeys, references []interface{} + for _, field := range constraint.ForeignKeys { + foreignKeys = append(foreignKeys, clause.Column{Name: field.DBName}) + } + + for _, field := range constraint.References { + references = append(references, clause.Column{Name: field.DBName}) + } + results = append(results, clause.Table{Name: constraint.Name}, foreignKeys, clause.Table{Name: constraint.ReferenceSchema.Table}, references) + return +} + +func (m Migrator) getRawDDL(table string) (string, error) { + var createSQL string + m.DB.Raw("SELECT sql FROM sqlite_master WHERE type = ? AND tbl_name = ? AND name = ?", "table", table, table).Row().Scan(&createSQL) + + if m.DB.Error != nil { + return "", m.DB.Error + } + return createSQL, nil +} + +func (m Migrator) recreateTable(value interface{}, tablePtr *string, + getCreateSQL func(rawDDL string, stmt *gorm.Statement) (sql string, sqlArgs []interface{}, err error)) error { + return m.RunWithValue(value, func(stmt *gorm.Statement) error { + table := stmt.Table + if tablePtr != nil { + table = *tablePtr + } + + rawDDL, err := m.getRawDDL(table) + if err != nil { + return err + } + + newTableName := table + "__temp" + + createSQL, sqlArgs, err := getCreateSQL(rawDDL, stmt) + if err != nil { + return err + } + if createSQL == "" { + return nil + } + + tableReg, err := regexp.Compile(" ('|`|\"| )" + table + "('|`|\"| ) ") + if err != nil { + return err + } + createSQL = tableReg.ReplaceAllString(createSQL, fmt.Sprintf(" `%v` ", newTableName)) + + createDDL, err := parseDDL(createSQL) + if err != nil { + return err + } + columns := createDDL.getColumns() + + return m.DB.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec(createSQL, sqlArgs...).Error; err != nil { + return err + } + + queries := []string{ + fmt.Sprintf("INSERT INTO `%v`(%v) SELECT %v FROM `%v`", newTableName, strings.Join(columns, ","), strings.Join(columns, ","), table), + fmt.Sprintf("DROP TABLE `%v`", table), + fmt.Sprintf("ALTER TABLE `%v` RENAME TO `%v`", newTableName, table), + } + for _, query := range queries { + if err := tx.Exec(query).Error; err != nil { + return err + } + } + return nil + }) + }) +} diff --git a/pkg/gorm-modernc-sqlite/sqlite.go b/pkg/gorm-modernc-sqlite/sqlite.go new file mode 100644 index 00000000..6e856ea3 --- /dev/null +++ b/pkg/gorm-modernc-sqlite/sqlite.go @@ -0,0 +1,219 @@ +package sqlite + +import ( + "context" + "database/sql" + "strconv" + "strings" + + "gorm.io/gorm/callbacks" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" + "gorm.io/gorm/migrator" + "gorm.io/gorm/schema" + + _ "modernc.org/sqlite" +) + +// DriverName is the default driver name for SQLite. +const DriverName = "sqlite" + +type Dialector struct { + DriverName string + DSN string + Conn gorm.ConnPool +} + +func Open(dsn string) gorm.Dialector { + return &Dialector{DSN: dsn} +} + +func (dialector Dialector) Name() string { + return "sqlite" +} + +func (dialector Dialector) Initialize(db *gorm.DB) (err error) { + if dialector.DriverName == "" { + dialector.DriverName = DriverName + } + + if dialector.Conn != nil { + db.ConnPool = dialector.Conn + } else { + conn, err := sql.Open(dialector.DriverName, dialector.DSN) + if err != nil { + return err + } + db.ConnPool = conn + } + + var version string + if err := db.ConnPool.QueryRowContext(context.Background(), "select sqlite_version()").Scan(&version); err != nil { + return err + } + // https://www.sqlite.org/releaselog/3_35_0.html + if compareVersion(version, "3.35.0") >= 0 { + callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{ + CreateClauses: []string{"INSERT", "VALUES", "ON CONFLICT", "RETURNING"}, + UpdateClauses: []string{"UPDATE", "SET", "WHERE", "RETURNING"}, + DeleteClauses: []string{"DELETE", "FROM", "WHERE", "RETURNING"}, + LastInsertIDReversed: true, + }) + } else { + callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{ + LastInsertIDReversed: true, + }) + } + + for k, v := range dialector.ClauseBuilders() { + db.ClauseBuilders[k] = v + } + return +} + +func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder { + return map[string]clause.ClauseBuilder{ + "INSERT": func(c clause.Clause, builder clause.Builder) { + if insert, ok := c.Expression.(clause.Insert); ok { + if stmt, ok := builder.(*gorm.Statement); ok { + stmt.WriteString("INSERT ") + if insert.Modifier != "" { + stmt.WriteString(insert.Modifier) + stmt.WriteByte(' ') + } + + stmt.WriteString("INTO ") + if insert.Table.Name == "" { + stmt.WriteQuoted(stmt.Table) + } else { + stmt.WriteQuoted(insert.Table) + } + return + } + } + + c.Build(builder) + }, + "LIMIT": func(c clause.Clause, builder clause.Builder) { + if limit, ok := c.Expression.(clause.Limit); ok { + if limit.Limit > 0 || limit.Offset > 0 { + if limit.Limit <= 0 { + limit.Limit = -1 + } + builder.WriteString("LIMIT " + strconv.Itoa(limit.Limit)) + } + if limit.Offset > 0 { + builder.WriteString(" OFFSET " + strconv.Itoa(limit.Offset)) + } + } + }, + "FOR": func(c clause.Clause, builder clause.Builder) { + if _, ok := c.Expression.(clause.Locking); ok { + // SQLite3 does not support row-level locking. + return + } + c.Build(builder) + }, + } +} + +func (dialector Dialector) DefaultValueOf(field *schema.Field) clause.Expression { + if field.AutoIncrement { + return clause.Expr{SQL: "NULL"} + } + + // doesn't work, will raise error + return clause.Expr{SQL: "DEFAULT"} +} + +func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator { + return Migrator{migrator.Migrator{Config: migrator.Config{ + DB: db, + Dialector: dialector, + CreateIndexAfterCreateTable: true, + }}} +} + +func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) { + writer.WriteByte('?') +} + +func (dialector Dialector) QuoteTo(writer clause.Writer, str string) { + writer.WriteByte('`') + if strings.Contains(str, ".") { + for idx, str := range strings.Split(str, ".") { + if idx > 0 { + writer.WriteString(".`") + } + writer.WriteString(str) + writer.WriteByte('`') + } + } else { + writer.WriteString(str) + writer.WriteByte('`') + } +} + +func (dialector Dialector) Explain(sql string, vars ...interface{}) string { + return logger.ExplainSQL(sql, nil, `"`, vars...) +} + +func (dialector Dialector) DataTypeOf(field *schema.Field) string { + switch field.DataType { + case schema.Bool: + return "numeric" + case schema.Int, schema.Uint: + if field.AutoIncrement && !field.PrimaryKey { + // https://www.sqlite.org/autoinc.html + return "integer PRIMARY KEY AUTOINCREMENT" + } else { + return "integer" + } + case schema.Float: + return "real" + case schema.String: + return "text" + case schema.Time: + return "datetime" + case schema.Bytes: + return "blob" + } + + return string(field.DataType) +} + +func (dialectopr Dialector) SavePoint(tx *gorm.DB, name string) error { + tx.Exec("SAVEPOINT " + name) + return nil +} + +func (dialectopr Dialector) RollbackTo(tx *gorm.DB, name string) error { + tx.Exec("ROLLBACK TO SAVEPOINT " + name) + return nil +} + +func compareVersion(version1, version2 string) int { + n, m := len(version1), len(version2) + i, j := 0, 0 + for i < n || j < m { + x := 0 + for ; i < n && version1[i] != '.'; i++ { + x = x*10 + int(version1[i]-'0') + } + i++ + y := 0 + for ; j < m && version2[j] != '.'; j++ { + y = y*10 + int(version2[j]-'0') + } + j++ + if x > y { + return 1 + } + if x < y { + return -1 + } + } + return 0 +} diff --git a/pkg/gorm-modernc-sqlite/sqlite_test.go b/pkg/gorm-modernc-sqlite/sqlite_test.go new file mode 100644 index 00000000..138646a1 --- /dev/null +++ b/pkg/gorm-modernc-sqlite/sqlite_test.go @@ -0,0 +1,81 @@ +package sqlite + +import ( + "fmt" + "testing" + + "gorm.io/gorm" + _ "modernc.org/sqlite" +) + +func TestDialector(t *testing.T) { + // This is the DSN of the in-memory SQLite database for these tests. + const InMemoryDSN = "file:testdatabase?mode=memory&cache=shared" + + rows := []struct { + description string + dialector *Dialector + openSuccess bool + query string + querySuccess bool + }{ + { + description: "Default driver", + dialector: &Dialector{ + DSN: InMemoryDSN, + }, + openSuccess: true, + query: "SELECT 1", + querySuccess: true, + }, + { + description: "Explicit default driver", + dialector: &Dialector{ + DriverName: DriverName, + DSN: InMemoryDSN, + }, + openSuccess: true, + query: "SELECT 1", + querySuccess: true, + }, + { + description: "Bad driver", + dialector: &Dialector{ + DriverName: "not-a-real-driver", + DSN: InMemoryDSN, + }, + openSuccess: false, + }, + } + for rowIndex, row := range rows { + t.Run(fmt.Sprintf("%d/%s", rowIndex, row.description), func(t *testing.T) { + db, err := gorm.Open(row.dialector, &gorm.Config{}) + if !row.openSuccess { + if err == nil { + t.Errorf("Expected Open to fail.") + } + return + } + + if err != nil { + t.Errorf("Expected Open to succeed; got error: %v", err) + } + if db == nil { + t.Errorf("Expected db to be non-nil.") + } + if row.query != "" { + err = db.Exec(row.query).Error + if !row.querySuccess { + if err == nil { + t.Errorf("Expected query to fail.") + } + return + } + + if err != nil { + t.Errorf("Expected query to succeed; got error: %v", err) + } + } + }) + } +}