From cbd98708254c75cad66ed20b72e3c3aa0f674dd5 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 5 Nov 2025 13:29:15 +0100 Subject: [PATCH] replace bun native mysql code with drizzle orm and mysql2 as driver --- dlt-connector/bun.lock | 34 +++- dlt-connector/package.json | 2 + .../db-v2.7.0_to_blockchain-v3.5/Context.ts | 26 +-- .../db-v2.7.0_to_blockchain-v3.5/bootstrap.ts | 2 +- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 158 ++++++++---------- .../drizzle.schema.ts | 53 ++++++ .../db-v2.7.0_to_blockchain-v3.5/index.ts | 14 +- 7 files changed, 186 insertions(+), 103 deletions(-) create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 7ad790bf3..f4dcee45b 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -17,11 +17,13 @@ "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", "dotenv": "^10.0.0", + "drizzle-orm": "^0.44.7", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", "jsonrpc-ts-client": "^0.2.3", "log4js": "^6.9.1", + "mysql2": "^3.15.3", "typescript": "^5.8.3", "uuid": "^8.3.2", "valibot": "1.1.0", @@ -322,6 +324,8 @@ "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="], + "aws-ssl-profiles": ["aws-ssl-profiles@1.1.2", "", {}, "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g=="], + "axios": ["axios@0.24.0", "", { "dependencies": { "follow-redirects": "^1.14.4" } }, "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA=="], "babel-jest": ["babel-jest@29.7.0", "", { "dependencies": { "@jest/transform": "^29.7.0", "@types/babel__core": "^7.1.14", "babel-plugin-istanbul": "^6.1.1", "babel-preset-jest": "^29.6.3", "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "slash": "^3.0.0" }, "peerDependencies": { "@babel/core": "^7.8.0" } }, "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg=="], @@ -426,12 +430,16 @@ "delegates": ["delegates@1.0.0", "", {}, "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ=="], + "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], "destroy": ["destroy@1.2.0", "", {}, "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="], "dotenv": ["dotenv@10.0.0", "", {}, "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q=="], + "drizzle-orm": ["drizzle-orm@0.44.7", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-quIpnYznjU9lHshEOAYLoZ9s3jweleHlZIAWR/jX9gAWNg/JhQ1wj0KGRf7/Zm+obRrYd9GjPVJg790QY9N5AQ=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], @@ -530,6 +538,8 @@ "gauge": ["gauge@4.0.4", "", { "dependencies": { "aproba": "^1.0.3 || ^2.0.0", "color-support": "^1.1.3", "console-control-strings": "^1.1.0", "has-unicode": "^2.0.1", "signal-exit": "^3.0.7", "string-width": "^4.2.3", "strip-ansi": "^6.0.1", "wide-align": "^1.1.5" } }, "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg=="], + "generate-function": ["generate-function@2.3.1", "", { "dependencies": { "is-property": "^1.0.2" } }, "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ=="], + "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], "get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="], @@ -578,6 +588,8 @@ "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + "iconv-lite": ["iconv-lite@0.7.0", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ=="], + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], "image-size": ["image-size@1.2.1", "", { "dependencies": { "queue": "6.0.2" }, "bin": { "image-size": "bin/image-size.js" } }, "sha512-rH+46sQJ2dlwfjfhCyNx5thzrv+dtmBIhPHk0zgRUukHzZ/kRueTJXoYYsclBaKcSMBWuGbOFXtioLpzTb5euw=="], @@ -598,6 +610,8 @@ "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], + "is-property": ["is-property@1.0.2", "", {}, "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g=="], + "is-wsl": ["is-wsl@2.2.0", "", { "dependencies": { "is-docker": "^2.0.0" } }, "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww=="], "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], @@ -662,7 +676,9 @@ "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], - "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], + "lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="], + + "lru.min": ["lru.min@1.1.2", "", {}, "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg=="], "makeerror": ["makeerror@1.0.12", "", { "dependencies": { "tmpl": "1.0.5" } }, "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg=="], @@ -730,6 +746,10 @@ "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "mysql2": ["mysql2@3.15.3", "", { "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.7.0", "long": "^5.2.1", "lru.min": "^1.0.0", "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" } }, "sha512-FBrGau0IXmuqg4haEZRBfHNWB5mUARw6hNwPDXXGg0XzVJ50mr/9hb267lvpVMnhZ1FON3qNd4Xfcez1rbFwSg=="], + + "named-placeholders": ["named-placeholders@1.1.3", "", { "dependencies": { "lru-cache": "^7.14.1" } }, "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w=="], + "nan": ["nan@2.23.0", "", {}, "sha512-1UxuyYGdoQHcGg87Lkqm3FzefucTa0NAiOcuRsDmysep3c1LVCRK2krrUDafMWtjSG04htvAmvg96+SDknOmgQ=="], "negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="], @@ -856,6 +876,8 @@ "safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="], + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + "scheduler": ["scheduler@0.26.0", "", {}, "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA=="], "secure-json-parse": ["secure-json-parse@2.7.0", "", {}, "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="], @@ -864,6 +886,8 @@ "send": ["send@0.19.0", "", { "dependencies": { "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", "http-errors": "2.0.0", "mime": "1.6.0", "ms": "2.1.3", "on-finished": "2.4.1", "range-parser": "~1.2.1", "statuses": "2.0.1" } }, "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw=="], + "seq-queue": ["seq-queue@0.0.5", "", {}, "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q=="], + "serialize-error": ["serialize-error@2.1.0", "", {}, "sha512-ghgmKt5o4Tly5yEG/UJp8qTd0AN7Xalw4XBtDEKP655B699qMEtra1WlXeE6WIvdEG481JvRxULKsInq/iNysw=="], "serve-static": ["serve-static@1.16.2", "", { "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", "send": "0.19.0" } }, "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw=="], @@ -894,6 +918,8 @@ "sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="], + "sqlstring": ["sqlstring@2.3.3", "", {}, "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg=="], + "stack-utils": ["stack-utils@2.0.6", "", { "dependencies": { "escape-string-regexp": "^2.0.0" } }, "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ=="], "stackframe": ["stackframe@1.3.4", "", {}, "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw=="], @@ -1008,6 +1034,8 @@ "@babel/core/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "@babel/helper-compilation-targets/lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], + "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "@babel/traverse/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -1072,8 +1100,6 @@ "log4js/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - "lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], - "metro/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "metro/ws": ["ws@7.5.10", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": "^5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ=="], @@ -1118,6 +1144,8 @@ "wrap-ansi/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + "@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], + "cliui/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "cmake-js/fs-extra/jsonfile": ["jsonfile@6.2.0", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index b314584c6..749258400 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -31,11 +31,13 @@ "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", "dotenv": "^10.0.0", + "drizzle-orm": "^0.44.7", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", "jsonrpc-ts-client": "^0.2.3", "log4js": "^6.9.1", + "mysql2": "^3.15.3", "typescript": "^5.8.3", "uuid": "^8.3.2", "valibot": "1.1.0", diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts index dde47b7c4..5708cccd3 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts @@ -1,4 +1,5 @@ -import { SQL } from 'bun' +import { drizzle, MySql2Database } from 'drizzle-orm/mysql2' +import mysql from 'mysql2/promise' import { InMemoryBlockchain } from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' @@ -17,29 +18,30 @@ export type CommunityContext = { export class Context { public logger: Logger - public db: SQL + public db: MySql2Database public communities: Map public cache: KeyPairCacheManager - constructor(logger: Logger, db: SQL, cache: KeyPairCacheManager) { + constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager) { this.logger = logger this.db = db this.cache = cache this.communities = new Map() } - static create(): Context { + static async create(): Promise { loadConfig() + + const connection = await mysql.createConnection({ + host: CONFIG.MYSQL_HOST, + user: CONFIG.MYSQL_USER, + password: CONFIG.MYSQL_PASSWORD, + database: CONFIG.MYSQL_DATABASE, + port: CONFIG.MYSQL_PORT + }) return new Context( getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`), - new SQL({ - adapter: 'mysql', - hostname: CONFIG.MYSQL_HOST, - username: CONFIG.MYSQL_USER, - password: CONFIG.MYSQL_PASSWORD, - database: CONFIG.MYSQL_DATABASE, - port: CONFIG.MYSQL_PORT - }), + drizzle({ client: connection }), KeyPairCacheManager.getInstance() ) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts index 325347369..f987d579e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts @@ -9,7 +9,7 @@ import { communityDbToCommunity } from './convert' import { addCommunityRootTransaction } from './blockchain' export async function bootstrap(): Promise { - const context = Context.create() + const context = await Context.create() context.communities = await bootstrapCommunities(context) return context } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts index f63a7a002..125c9fb71 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts @@ -1,10 +1,13 @@ -import { SQL } from 'bun' import { memoSchema, uuidv4Schema, identifierSeedSchema, gradidoAmountSchema } from '../../schemas/typeGuard.schema' import { dateSchema, booleanSchema } from '../../schemas/typeConverter.schema' import * as v from 'valibot' -import { GradidoUnit } from 'gradido-blockchain-js' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { getLogger } from 'log4js' +import { MySql2Database } from 'drizzle-orm/mysql2' +import { communitiesTable, transactionLinksTable, transactionsTable, usersTable } from './drizzle.schema' +import { asc, sql, eq, isNotNull } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { GradidoUnit } from 'gradido-blockchain-js' export const createdUserDbSchema = v.object({ gradidoId: uuidv4Schema, @@ -64,14 +67,19 @@ export type CommunityDb = v.InferOutput const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`) // queries -export async function loadCommunities(db: SQL): Promise { - const result = await db` - SELECT c.foreign, c.community_uuid as communityUuid, c.name as name, c.creation_date as creationDate, MIN(u.created_at) as userMinCreatedAt - FROM communities c - LEFT JOIN users u ON c.community_uuid = u.community_uuid - WHERE c.community_uuid IS NOT NULL - GROUP BY c.community_uuid - ` +export async function loadCommunities(db: MySql2Database): Promise { + const result = await db.select({ + foreign: communitiesTable.foreign, + communityUuid: communitiesTable.communityUuid, + name: communitiesTable.name, + creationDate: communitiesTable.creationDate, + userMinCreatedAt: sql`MIN(${usersTable.createdAt})`, + }) + .from(communitiesTable) + .leftJoin(usersTable, eq(communitiesTable.communityUuid, usersTable.communityUuid)) + .where(isNotNull(communitiesTable.communityUuid)) + .groupBy(communitiesTable.communityUuid) + const communityNames = new Set() return result.map((row: any) => { let alias = row.name @@ -87,36 +95,39 @@ export async function loadCommunities(db: SQL): Promise { }) } -export async function loadUsers(db: SQL, offset: number, count: number): Promise { - const result = await db` - SELECT gradido_id as gradidoId, community_uuid as communityUuid, created_at as createdAt FROM users - ORDER by created_at ASC - LIMIT ${offset}, ${count} - ` +export async function loadUsers(db: MySql2Database, offset: number, count: number): Promise { + const result = await db.select() + .from(usersTable) + .orderBy(asc(usersTable.createdAt)) + .limit(count).offset(offset) + return result.map((row: any) => { return v.parse(createdUserDbSchema, row) }) } -export async function loadTransactions(db: SQL, offset: number, count: number): Promise { - const result = await db` - SELECT t.id, t.type_id, t.amount, t.balance_date, t.memo, t.creation_date, - u.gradido_id AS user_gradido_id, u.community_uuid AS user_community_uuid, u.created_at as user_created_at, - lu.gradido_id AS linked_user_gradido_id, lu.community_uuid AS linked_user_community_uuid, lu.created_at as linked_user_created_at, - tl.code as transaction_link_code - FROM transactions as t - LEFT JOIN users u ON t.user_id = u.id - LEFT JOIN users lu ON t.linked_user_id = lu.id - LEFT JOIN transaction_links tl ON t.transaction_link_id = tl.id - ORDER by balance_date ASC - LIMIT ${offset}, ${count} - ` +export async function loadTransactions(db: MySql2Database, offset: number, count: number): Promise { + const linkedUsers = alias(usersTable, 'linkedUser') + + const result = await db.select({ + transaction: transactionsTable, + user: usersTable, + linkedUser: linkedUsers, + transactionLink: transactionLinksTable, + }) + .from(transactionsTable) + .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) + .leftJoin(transactionLinksTable, eq(transactionsTable.transactionLinkId, transactionLinksTable.id)) + .orderBy(asc(transactionsTable.balanceDate)) + .limit(count).offset(offset) + return result.map((row: any) => { // console.log(row) // check for consistent data beforehand - const userCreatedAt = new Date(row.user_created_at) - const linkedUserCreatedAd = new Date(row.linked_user_created_at) - const balanceDate = new Date(row.balance_date) + const userCreatedAt = new Date(row.user.createdAt) + const linkedUserCreatedAd = new Date(row.linkedUser.createdAt) + const balanceDate = new Date(row.transaction.balanceDate) if (userCreatedAt.getTime() > balanceDate.getTime() || linkedUserCreatedAd.getTime() > balanceDate.getTime() ){ @@ -124,79 +135,58 @@ export async function loadTransactions(db: SQL, offset: number, count: number): throw new Error('at least one user was created after transaction balance date, logic error!') } - let amount = GradidoUnit.fromString(row.amount) - if (row.type_id === TransactionTypeId.SEND) { + let amount = GradidoUnit.fromString(row.transaction.amount) + if (row.transaction.typeId === TransactionTypeId.SEND) { amount = amount.mul(new GradidoUnit(-1)) } try { return v.parse(transactionDbSchema, { - typeId: row.type_id, - amount, - balanceDate: new Date(row.balance_date), - memo: row.memo, - creationDate: new Date(row.creation_date), - transactionLinkCode: row.transaction_link_code, - user: { - gradidoId: row.user_gradido_id, - communityUuid: row.user_community_uuid - }, - linkedUser: { - gradidoId: row.linked_user_gradido_id, - communityUuid: row.linked_user_community_uuid - } + ...row.transaction, + transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, + user: row.user, + linkedUser: row.linkedUser, }) } catch (e) { if (e instanceof v.ValiError) { - console.error(v.flatten(e.issues)) - } else { - throw e + logger.error(v.flatten(e.issues)) } + throw e } }) } -export async function loadTransactionLinks(db: SQL, offset: number, count: number): Promise { - const result = await db` - SELECT u.gradido_id as userGradidoId, u.community_uuid as userCommunityUuid, tl.code, tl.amount, tl.memo, tl.createdAt, tl.validUntil - FROM transaction_links tl - LEFT JOIN users u ON tl.userId = u.id - ORDER by createdAt ASC - LIMIT ${offset}, ${count} - ` +export async function loadTransactionLinks(db: MySql2Database, offset: number, count: number): Promise { + const result = await db.select() + .from(transactionLinksTable) + .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .orderBy(asc(transactionLinksTable.createdAt)) + .limit(count).offset(offset) + return result.map((row: any) => { return v.parse(transactionLinkDbSchema, { - ...row, - user: { - gradidoId: row.userGradidoId, - communityUuid: row.userCommunityUuid - } + ...row.transaction_links, + user: row.users, }) }) } -export async function loadDeletedTransactionLinks(db: SQL, offset: number, count: number): Promise { - const result = await db` - SELECT u.gradido_id as user_gradido_id, u.community_uuid as user_community_uuid, - tl.code, tl.amount, tl.memo, tl.deletedAt - FROM transaction_links tl - LEFT JOIN users u ON tl.userId = u.id - WHERE deletedAt IS NOT NULL - ORDER by deletedAt ASC - LIMIT ${offset}, ${count} - ` +export async function loadDeletedTransactionLinks(db: MySql2Database, offset: number, count: number): Promise { + const result = await db.select() + .from(transactionLinksTable) + .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .where(isNotNull(transactionLinksTable.deletedAt)) + .orderBy(asc(transactionLinksTable.deletedAt)) + .limit(count).offset(offset) + return result.map((row: any) => { - const user = { - gradidoId: row.user_gradido_id, - communityUuid: row.user_community_uuid - } return v.parse(transactionDbSchema, { typeId: TransactionTypeId.RECEIVE, - amount: row.amount, - balanceDate: new Date(row.deletedAt), - memo: row.memo, - transactionLinkCode: row.code, - user, - linkedUser: user + amount: row.transaction_links.amount, + balanceDate: new Date(row.transaction_links.deletedAt), + memo: row.transaction_links.memo, + transactionLinkCode: row.transaction_links.code, + user: row.users, + linkedUser: row.users }) }) } \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts new file mode 100644 index 000000000..12c92eaa9 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -0,0 +1,53 @@ + +import { mysqlTable, unique, int, varchar, char, datetime, tinyint, decimal, index } from 'drizzle-orm/mysql-core' +import { sql } from 'drizzle-orm' + + +// use only fields needed in the migration, after update the rest of the project, import database instead +export const communitiesTable = mysqlTable('communities', { + foreign: tinyint().default(1).notNull(), + communityUuid: char('community_uuid', { length: 36 }).default(sql`NULL`), + name: varchar({ length: 40 }).default(sql`NULL`), + creationDate: datetime('creation_date', { mode: 'string', fsp: 3 }).default(sql`NULL`), +}, +(table) => [ + unique('uuid_key').on(table.communityUuid), +]) + + +export const usersTable = mysqlTable('users', { + id: int().autoincrement().notNull(), + gradidoId: char('gradido_id', { length: 36 }).notNull(), + communityUuid: varchar('community_uuid', { length: 36 }).default(sql`NULL`), + createdAt: datetime('created_at', { mode: 'string', fsp: 3 }).default(sql`current_timestamp(3)`).notNull(), +}, +(table) => [ + unique('uuid_key').on(table.gradidoId, table.communityUuid), +]) + +export const transactionsTable = mysqlTable('transactions', { + id: int().autoincrement().notNull(), + typeId: int("type_id").default(sql`NULL`), + transactionLinkId: int("transaction_link_id").default(sql`NULL`), + amount: decimal({ precision: 40, scale: 20 }).default(sql`NULL`), + balanceDate: datetime("balance_date", { mode: 'string', fsp: 3 }).default(sql`current_timestamp(3)`).notNull(), + memo: varchar({ length: 255 }).notNull(), + creationDate: datetime("creation_date", { mode: 'string', fsp: 3 }).default(sql`NULL`), + userId: int("user_id").notNull(), + linkedUserId: int("linked_user_id").default(sql`NULL`), +}, +(table) => [ + index("user_id").on(table.userId), +]) + + +export const transactionLinksTable = mysqlTable('transaction_links', { + id: int().autoincrement().notNull(), + userId: int().notNull(), + amount: decimal({ precision: 40, scale: 20 }).notNull(), + memo: varchar({ length: 255 }).notNull(), + code: varchar({ length: 24 }).notNull(), + createdAt: datetime({ mode: 'string'}).notNull(), + deletedAt: datetime({ mode: 'string'}).default(sql`NULL`), + validUntil: datetime({ mode: 'string'}).notNull(), +}) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 8d66db750..1c309d021 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -5,8 +5,15 @@ import { Profiler } from 'gradido-blockchain-js' import { Logger } from 'log4js' - -import { CreatedUserDb, loadDeletedTransactionLinks, loadTransactionLinks, loadTransactions, loadUsers, TransactionDb, TransactionLinkDb } from './database' +import { + CreatedUserDb, + loadDeletedTransactionLinks, + loadTransactionLinks, + loadTransactions, + loadUsers, + TransactionDb, + TransactionLinkDb +} from './database' import { addRegisterAddressTransaction, addTransaction } from './blockchain' import { generateKeyPairUserAccount } from './keyPair' import { transactionDbToTransaction, transactionLinkDbToTransaction, userDbToTransaction } from './convert' @@ -28,7 +35,6 @@ async function main() { if(error) { context.logger.error(error) } - context.db.close() }) // synchronize to blockchain @@ -39,6 +45,7 @@ async function main() { (user: CreatedUserDb) => user.createdAt, (context: Context, user: CreatedUserDb) => pushRegisterAddressTransaction(context, user) ) + const transactions = new OrderedContainer( getNextTransactions, (transaction: TransactionDb) => transaction.balanceDate, @@ -56,6 +63,7 @@ async function main() { (transaction: TransactionDb) => transaction.balanceDate, (context: Context, transaction: TransactionDb) => pushTransaction(context, transaction) ) + try { await synchronizeToBlockchain(context, [users, transactions, transactionLinks, deletedTransactionLinks], BATCH_SIZE) } catch (e) {