From 4834c01fe9f32420282a98b14b6301324415e5a3 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 24 Apr 2025 11:57:19 +0200 Subject: [PATCH 01/21] prevent overwriting process message with error message --- backend/src/apis/humhub/ExportUsers.ts | 35 ++++++++++++++++---------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/backend/src/apis/humhub/ExportUsers.ts b/backend/src/apis/humhub/ExportUsers.ts index b3af27f7d..5214fef5d 100644 --- a/backend/src/apis/humhub/ExportUsers.ts +++ b/backend/src/apis/humhub/ExportUsers.ts @@ -34,6 +34,7 @@ async function loadUsersFromHumHub(client: HumHubClient): Promise[] = [] - users.forEach((user: User) => promises.push(syncUser(user, humhubUsers))) - const executedActions = await Promise.all(promises) - executedActions.forEach((executedAction: ExecutedHumhubAction) => { - executedHumhubActionsCount[executedAction as number]++ - }) - // using process.stdout.write here so that carriage-return is working analog to c - // printf("\rchecked user: %d/%d", dbUserCount, totalUsers); - process.stdout.write(`checked user: ${dbUserCount}/${totalUsers}\r`) + try { + const [users, totalUsers] = await getUsersPage(page, USER_BULK_SIZE) + dbUserCount += users.length + userCount = users.length + page++ + const promises: Promise[] = [] + users.forEach((user: User) => promises.push(syncUser(user, humhubUsers))) + const executedActions = await Promise.all(promises) + executedActions.forEach((executedAction: ExecutedHumhubAction) => { + executedHumhubActionsCount[executedAction as number]++ + }) + // using process.stdout.write here so that carriage-return is working analog to c + // printf("\rchecked user: %d/%d", dbUserCount, totalUsers); + process.stdout.write(`checked user: ${dbUserCount}/${totalUsers}\r`) + } catch (e) { + process.stdout.write('\n') + throw e + } } while (userCount === USER_BULK_SIZE) + process.stdout.write('\n') await con.destroy() const elapsed = new Date().getTime() - start From 4860cc2670e89e95f09ab20253a8369325854dad Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 24 Apr 2025 13:05:16 +0200 Subject: [PATCH 02/21] use old humhub username on update for finding humhub user to prevent new user if user has changed email in humhub and alias in wallet --- backend/src/apis/humhub/ExportUsers.ts | 4 ++-- backend/src/apis/humhub/__mocks__/syncUser.ts | 4 +++- backend/src/apis/humhub/syncUser.test.ts | 7 ++++--- backend/src/apis/humhub/syncUser.ts | 7 +++---- backend/src/graphql/resolver/UserResolver.ts | 20 ++++++++++++++----- .../graphql/resolver/util/syncHumhub.test.ts | 4 ++-- .../src/graphql/resolver/util/syncHumhub.ts | 13 ++++++------ backend/test/testSetup.ts | 2 ++ 8 files changed, 37 insertions(+), 24 deletions(-) diff --git a/backend/src/apis/humhub/ExportUsers.ts b/backend/src/apis/humhub/ExportUsers.ts index 5214fef5d..4ed6f2928 100644 --- a/backend/src/apis/humhub/ExportUsers.ts +++ b/backend/src/apis/humhub/ExportUsers.ts @@ -26,7 +26,7 @@ function getUsersPage(page: number, limit: number): Promise<[User[], number]> { /** * @param client - * @returns user map indices with email + * @returns user map indices with username */ async function loadUsersFromHumHub(client: HumHubClient): Promise> { const start = new Date().getTime() @@ -43,7 +43,7 @@ async function loadUsersFromHumHub(client: HumHubClient): Promise { // deleted users have empty emails if (user.account.email) { - humhubUsers.set(user.account.email.trim(), user) + humhubUsers.set(user.account.username, user) } else { skippedUsersCount++ } diff --git a/backend/src/apis/humhub/__mocks__/syncUser.ts b/backend/src/apis/humhub/__mocks__/syncUser.ts index 7e0660da4..1cb2edfbf 100644 --- a/backend/src/apis/humhub/__mocks__/syncUser.ts +++ b/backend/src/apis/humhub/__mocks__/syncUser.ts @@ -2,6 +2,7 @@ import { User } from '@entity/User' import { isHumhubUserIdenticalToDbUser } from '@/apis/humhub/compareHumhubUserDbUser' import { GetUser } from '@/apis/humhub/model/GetUser' +import { PostUser } from '@/apis/humhub/model/PostUser' export enum ExecutedHumhubAction { UPDATE, @@ -26,7 +27,8 @@ export async function syncUser( user: User, humhubUsers: Map, ): Promise { - const humhubUser = humhubUsers.get(user.emailContact.email.trim()) + const postUser = new PostUser(user) + const humhubUser = humhubUsers.get(postUser.account.username) if (humhubUser) { if (!user.humhubAllowed) { return Promise.resolve(ExecutedHumhubAction.DELETE) diff --git a/backend/src/apis/humhub/syncUser.test.ts b/backend/src/apis/humhub/syncUser.test.ts index 20a6b2c33..00c125632 100644 --- a/backend/src/apis/humhub/syncUser.test.ts +++ b/backend/src/apis/humhub/syncUser.test.ts @@ -28,7 +28,8 @@ describe('syncUser function', () => { it('When humhubUser exists and user.humhubAllowed is false, should return DELETE action', async () => { const humhubUsers = new Map() - humhubUsers.set(defaultUser.emailContact.email, new GetUser(defaultUser, 1)) + const humhubUser = new GetUser(defaultUser, 1) + humhubUsers.set(humhubUser.account.username, humhubUser) defaultUser.humhubAllowed = false const result = await syncUser(defaultUser, humhubUsers) @@ -39,8 +40,8 @@ describe('syncUser function', () => { it('When humhubUser exists and user.humhubAllowed is true and there are changes in user data, should return UPDATE action', async () => { const humhubUsers = new Map() const humhubUser = new GetUser(defaultUser, 1) + humhubUsers.set(humhubUser.account.username, humhubUser) humhubUser.account.username = 'test username' - humhubUsers.set(defaultUser.emailContact.email, humhubUser) defaultUser.humhubAllowed = true const result = await syncUser(defaultUser, humhubUsers) @@ -51,7 +52,7 @@ describe('syncUser function', () => { it('When humhubUser exists and user.humhubAllowed is true and there are no changes in user data, should return SKIP action', async () => { const humhubUsers = new Map() const humhubUser = new GetUser(defaultUser, 1) - humhubUsers.set(defaultUser.emailContact.email, humhubUser) + humhubUsers.set(humhubUser.account.username, humhubUser) defaultUser.humhubAllowed = true const result = await syncUser(defaultUser, humhubUsers) diff --git a/backend/src/apis/humhub/syncUser.ts b/backend/src/apis/humhub/syncUser.ts index fc6fcc99b..8057626af 100644 --- a/backend/src/apis/humhub/syncUser.ts +++ b/backend/src/apis/humhub/syncUser.ts @@ -21,9 +21,8 @@ export enum ExecutedHumhubAction { * | true | true | false | SKIP * | false | false | ignored | SKIP * | false | true | ignored | CREATE - * @param user - * @param humHubClient - * @param humhubUsers + * @param user user entity + * @param humhubUsers user map indices with username * @returns */ export async function syncUser( @@ -31,7 +30,7 @@ export async function syncUser( humhubUsers: Map, ): Promise { const postUser = new PostUser(user) - const humhubUser = humhubUsers.get(user.emailContact.email.trim()) + const humhubUser = humhubUsers.get(postUser.account.username) const humHubClient = HumHubClient.getInstance() if (!humHubClient) { throw new LogError('Error creating humhub client') diff --git a/backend/src/graphql/resolver/UserResolver.ts b/backend/src/graphql/resolver/UserResolver.ts index fea150338..66e6a546b 100644 --- a/backend/src/graphql/resolver/UserResolver.ts +++ b/backend/src/graphql/resolver/UserResolver.ts @@ -37,6 +37,7 @@ import { UpdateUserInfosArgs } from '@arg/UpdateUserInfosArgs' import { OptInType } from '@enum/OptInType' import { Order } from '@enum/Order' import { PasswordEncryptionType } from '@enum/PasswordEncryptionType' +import { PublishNameType } from '@enum/PublishNameType' import { UserContactType } from '@enum/UserContactType' import { SearchAdminUsersResult } from '@model/AdminUser' // import { Location } from '@model/Location' @@ -54,6 +55,7 @@ import { subscribe } from '@/apis/KlicktippController' import { encode } from '@/auth/JWT' import { RIGHTS } from '@/auth/RIGHTS' import { CONFIG } from '@/config' +import { PublishNameLogic } from '@/data/PublishName.logic' import { sendAccountActivationEmail, sendAccountMultiRegistrationEmail, @@ -246,12 +248,12 @@ export class UserResolver { try { const result = await humhubUserPromise user.humhubAllowed = result?.result?.account.status === 1 - if (user.humhubAllowed) { + if (user.humhubAllowed && result?.result?.account?.username) { let spaceId = null if (projectBranding) { spaceId = projectBranding.spaceId } - void syncHumhub(null, dbUser, spaceId) + await syncHumhub(null, dbUser, result.result.account.username, spaceId) } } catch (e) { logger.error("couldn't reach out to humhub, disable for now", e) @@ -448,7 +450,11 @@ export class UserResolver { if (projectBranding) { spaceId = projectBranding.spaceId } - void syncHumhub(null, dbUser, spaceId) + try { + await syncHumhub(null, dbUser, dbUser.gradidoID, spaceId) + } catch (e) { + logger.error("createUser: couldn't reach out to humhub, disable for now", e) + } } if (redeemCode) { @@ -659,6 +665,10 @@ export class UserResolver { ) const user = getUser(context) const updateUserInGMS = compareGmsRelevantUserSettings(user, updateUserInfosArgs) + const publishNameLogic = new PublishNameLogic(user) + const oldHumhubUsername = publishNameLogic.getUserIdentifier( + user.humhubPublishName as PublishNameType, + ) // try { if (firstName) { @@ -764,7 +774,7 @@ export class UserResolver { } try { if (CONFIG.HUMHUB_ACTIVE) { - await syncHumhub(updateUserInfosArgs, user) + await syncHumhub(updateUserInfosArgs, user, oldHumhubUsername) } } catch (e) { logger.error('error sync user with humhub', e) @@ -839,7 +849,7 @@ export class UserResolver { } const humhubUserAccount = new HumhubAccount(dbUser) const autoLoginUrlPromise = humhubClient.createAutoLoginUrl(humhubUserAccount.username, project) - const humhubUser = await syncHumhub(null, dbUser) + const humhubUser = await syncHumhub(null, dbUser, humhubUserAccount.username) if (!humhubUser) { throw new LogError("user don't exist (any longer) on humhub and couldn't be created") } diff --git a/backend/src/graphql/resolver/util/syncHumhub.test.ts b/backend/src/graphql/resolver/util/syncHumhub.test.ts index c25eb52a8..d8a61af23 100644 --- a/backend/src/graphql/resolver/util/syncHumhub.test.ts +++ b/backend/src/graphql/resolver/util/syncHumhub.test.ts @@ -32,7 +32,7 @@ describe('syncHumhub', () => { }) it('Should not sync if no relevant changes', async () => { - await syncHumhub(mockUpdateUserInfosArg, new User()) + await syncHumhub(mockUpdateUserInfosArg, new User(), 'username') expect(HumHubClient.getInstance).not.toBeCalled() // language logging from some other place expect(logger.debug).toBeCalledTimes(5) @@ -42,7 +42,7 @@ describe('syncHumhub', () => { it('Should retrieve user from humhub and sync if relevant changes', async () => { mockUpdateUserInfosArg.firstName = 'New' // Relevant changes mockUser.firstName = 'New' - await syncHumhub(mockUpdateUserInfosArg, mockUser) + await syncHumhub(mockUpdateUserInfosArg, mockUser, 'username') expect(logger.debug).toHaveBeenCalledTimes(8) // Four language logging calls, two debug calls in function, one for not syncing expect(logger.info).toHaveBeenLastCalledWith('finished sync user with humhub', { localId: mockUser.id, diff --git a/backend/src/graphql/resolver/util/syncHumhub.ts b/backend/src/graphql/resolver/util/syncHumhub.ts index 90500bbc5..23ba6d0dd 100644 --- a/backend/src/graphql/resolver/util/syncHumhub.ts +++ b/backend/src/graphql/resolver/util/syncHumhub.ts @@ -2,10 +2,9 @@ import { User } from '@entity/User' import { HumHubClient } from '@/apis/humhub/HumHubClient' import { GetUser } from '@/apis/humhub/model/GetUser' +import { PostUser } from '@/apis/humhub/model/PostUser' import { ExecutedHumhubAction, syncUser } from '@/apis/humhub/syncUser' -import { PublishNameLogic } from '@/data/PublishName.logic' import { UpdateUserInfosArgs } from '@/graphql/arg/UpdateUserInfosArgs' -import { PublishNameType } from '@/graphql/enum/PublishNameType' import { backendLogger as logger } from '@/server/logger' /** @@ -17,6 +16,7 @@ import { backendLogger as logger } from '@/server/logger' export async function syncHumhub( updateUserInfosArg: UpdateUserInfosArgs | null, user: User, + oldHumhubUsername: string, spaceId?: number | null, ): Promise { // check for humhub relevant changes @@ -38,15 +38,13 @@ export async function syncHumhub( return } logger.debug('retrieve user from humhub') - const userNameLogic = new PublishNameLogic(user) - const username = userNameLogic.getUserIdentifier(user.humhubPublishName as PublishNameType) - let humhubUser = await humhubClient.userByUsername(username) + let humhubUser = await humhubClient.userByUsername(oldHumhubUsername) if (!humhubUser) { humhubUser = await humhubClient.userByEmail(user.emailContact.email) } const humhubUsers = new Map() if (humhubUser) { - humhubUsers.set(user.emailContact.email, humhubUser) + humhubUsers.set(humhubUser.account.username, humhubUser) } logger.debug('update user at humhub') const result = await syncUser(user, humhubUsers) @@ -63,7 +61,8 @@ export async function syncHumhub( logger.debug(`user added to space ${spaceId}`) } if (result !== ExecutedHumhubAction.SKIP) { - return await humhubClient.userByUsername(username) + const getUser = new PostUser(user) + return await humhubClient.userByUsername(getUser.account.username) } return humhubUser } diff --git a/backend/test/testSetup.ts b/backend/test/testSetup.ts index 8cf4ff084..74021f0a3 100644 --- a/backend/test/testSetup.ts +++ b/backend/test/testSetup.ts @@ -6,6 +6,8 @@ import { backendLogger as logger } from '@/server/logger' CONFIG.EMAIL = true CONFIG.EMAIL_TEST_MODUS = false +CONFIG.HUMHUB_ACTIVE = false +CONFIG.GMS_ACTIVE = false jest.setTimeout(1000000) From 456725aeae08c8ff0c843e871d81efaac680df6e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 24 Apr 2025 15:09:37 +0200 Subject: [PATCH 03/21] remove unnecessary \n, better final error reporting --- backend/src/apis/humhub/ExportUsers.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/src/apis/humhub/ExportUsers.ts b/backend/src/apis/humhub/ExportUsers.ts index 4ed6f2928..99c4471e5 100644 --- a/backend/src/apis/humhub/ExportUsers.ts +++ b/backend/src/apis/humhub/ExportUsers.ts @@ -34,7 +34,6 @@ async function loadUsersFromHumHub(client: HumHubClient): Promise { // eslint-disable-next-line no-console - console.error(e) + console.error(JSON.stringify(e, null, 2)) // eslint-disable-next-line n/no-process-exit process.exit(1) }) From 21be12f3b0857b79883914cfa5efdf187781b9f9 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 24 Apr 2025 15:27:30 +0200 Subject: [PATCH 04/21] check firstname length for humhub --- backend/src/apis/humhub/ExportUsers.ts | 3 ++- backend/src/apis/humhub/syncUser.ts | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/src/apis/humhub/ExportUsers.ts b/backend/src/apis/humhub/ExportUsers.ts index 99c4471e5..d4515b3ce 100644 --- a/backend/src/apis/humhub/ExportUsers.ts +++ b/backend/src/apis/humhub/ExportUsers.ts @@ -88,7 +88,7 @@ async function main() { const humhubUsers = await loadUsersFromHumHub(humHubClient) let dbUserCount = 0 - const executedHumhubActionsCount = [0, 0, 0, 0] + const executedHumhubActionsCount = [0, 0, 0, 0, 0] do { try { @@ -121,6 +121,7 @@ async function main() { updatedCount: executedHumhubActionsCount[ExecutedHumhubAction.UPDATE], skippedCount: executedHumhubActionsCount[ExecutedHumhubAction.SKIP], deletedCount: executedHumhubActionsCount[ExecutedHumhubAction.DELETE], + validationErrorCount: executedHumhubActionsCount[ExecutedHumhubAction.VALIDATION_ERROR], }) } diff --git a/backend/src/apis/humhub/syncUser.ts b/backend/src/apis/humhub/syncUser.ts index 8057626af..8b7963b07 100644 --- a/backend/src/apis/humhub/syncUser.ts +++ b/backend/src/apis/humhub/syncUser.ts @@ -1,6 +1,7 @@ import { User } from '@entity/User' import { LogError } from '@/server/LogError' +import { backendLogger as logger } from '@/server/logger' import { isHumhubUserIdenticalToDbUser } from './compareHumhubUserDbUser' import { HumHubClient } from './HumHubClient' @@ -12,7 +13,18 @@ export enum ExecutedHumhubAction { CREATE, SKIP, DELETE, + VALIDATION_ERROR, } + +// todo: replace with full validation (schema) +function isValid(postUser: PostUser, userId: number): boolean { + if (postUser.profile.firstname.length > 20) { + logger.error('firstname too long for humhub, for user with id:', userId) + return false + } + return true +} + /** * Trigger action according to conditions * | User exist on humhub | export to humhub allowed | changes in user data | ACTION @@ -30,6 +42,9 @@ export async function syncUser( humhubUsers: Map, ): Promise { const postUser = new PostUser(user) + if (!isValid(postUser, user.id)) { + return ExecutedHumhubAction.VALIDATION_ERROR + } const humhubUser = humhubUsers.get(postUser.account.username) const humHubClient = HumHubClient.getInstance() if (!humHubClient) { From c16786cd208afaeb42a2205da65471ce9a74d3e7 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 24 Apr 2025 16:22:43 +0200 Subject: [PATCH 05/21] slice first_name and last_name after 20 character for humhub --- .../humhub/compareHumhubUserDbUser.test.ts | 18 ++++++++++++++++++ backend/src/apis/humhub/syncUser.ts | 4 ++++ backend/src/data/PublishName.logic.ts | 16 ++++++++++++---- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/backend/src/apis/humhub/compareHumhubUserDbUser.test.ts b/backend/src/apis/humhub/compareHumhubUserDbUser.test.ts index cc636d17a..a3d6c7044 100644 --- a/backend/src/apis/humhub/compareHumhubUserDbUser.test.ts +++ b/backend/src/apis/humhub/compareHumhubUserDbUser.test.ts @@ -1,4 +1,7 @@ /* eslint-disable prettier/prettier */ +// eslint-disable-next-line import/no-unassigned-import +import 'reflect-metadata' +import { PublishNameType } from '@/graphql/enum/PublishNameType' import { communityDbUser } from '@/util/communityUser' import { isHumhubUserIdenticalToDbUser } from './compareHumhubUserDbUser' @@ -13,6 +16,7 @@ describe('isHumhubUserIdenticalToDbUser', () => { defaultUser.alias = 'alias' defaultUser.emailContact.email = 'email@gmail.com' defaultUser.language = 'en' + defaultUser.gradidoID = 'gradidoID' }) it('Should return true because humhubUser was created from entity user', () => { @@ -21,6 +25,20 @@ describe('isHumhubUserIdenticalToDbUser', () => { expect(result).toBe(true) }) + it('Should return false, because last name differ because of publish name type', () => { + const humhubUser = new GetUser(defaultUser, 1) + defaultUser.humhubPublishName = PublishNameType.PUBLISH_NAME_FIRST + const result = isHumhubUserIdenticalToDbUser(humhubUser, defaultUser) + expect(result).toBe(false) + }) + + it('Should return true, even if alias is empty', () => { + defaultUser.alias = '' + const humhubUser = new GetUser(defaultUser, 1) + const result = isHumhubUserIdenticalToDbUser(humhubUser, defaultUser) + expect(result).toBe(true) + }) + it('Should return false because first name differ', () => { const humhubUser = new GetUser(defaultUser, 1) humhubUser.profile.firstname = 'changed first name' diff --git a/backend/src/apis/humhub/syncUser.ts b/backend/src/apis/humhub/syncUser.ts index 8b7963b07..e5b793ee3 100644 --- a/backend/src/apis/humhub/syncUser.ts +++ b/backend/src/apis/humhub/syncUser.ts @@ -22,6 +22,10 @@ function isValid(postUser: PostUser, userId: number): boolean { logger.error('firstname too long for humhub, for user with id:', userId) return false } + if (postUser.profile.lastname.length > 20) { + logger.error('lastname too long for humhub, for user with id:', userId) + return false + } return true } diff --git a/backend/src/data/PublishName.logic.ts b/backend/src/data/PublishName.logic.ts index a78627d49..b70a29bb3 100644 --- a/backend/src/data/PublishName.logic.ts +++ b/backend/src/data/PublishName.logic.ts @@ -35,12 +35,16 @@ export class PublishNameLogic { * @returns user.firstName for PUBLISH_NAME_FIRST, PUBLISH_NAME_FIRST_INITIAL or PUBLISH_NAME_FULL */ public getFirstName(publishNameType: PublishNameType): string { + let firstName = '' + if (this.user && typeof this.user.firstName === 'string') { + firstName = this.user.firstName + } return [ PublishNameType.PUBLISH_NAME_FIRST, PublishNameType.PUBLISH_NAME_FIRST_INITIAL, PublishNameType.PUBLISH_NAME_FULL, ].includes(publishNameType) - ? this.user.firstName + ? firstName.slice(0, 20) : '' } @@ -51,10 +55,14 @@ export class PublishNameLogic { * first initial from user.lastName for PUBLISH_NAME_FIRST_INITIAL */ public getLastName(publishNameType: PublishNameType): string { + let lastName = '' + if (this.user && typeof this.user.lastName === 'string') { + lastName = this.user.lastName + } return publishNameType === PublishNameType.PUBLISH_NAME_FULL - ? this.user.lastName - : publishNameType === PublishNameType.PUBLISH_NAME_FIRST_INITIAL - ? this.user.lastName.charAt(0) + ? lastName.slice(0, 20) + : publishNameType === PublishNameType.PUBLISH_NAME_FIRST_INITIAL && lastName.length > 0 + ? lastName.charAt(0) : '' } From adf9e04c4f2e4571fadef51d1e78e13454c4e9b9 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 24 Apr 2025 16:42:42 +0200 Subject: [PATCH 06/21] request max 25 admins and mods on frontend information page --- frontend/src/graphql/queries.js | 4 ++-- frontend/src/pages/InfoStatistic.vue | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/graphql/queries.js b/frontend/src/graphql/queries.js index 65f7e9418..edd034344 100644 --- a/frontend/src/graphql/queries.js +++ b/frontend/src/graphql/queries.js @@ -256,8 +256,8 @@ export const communityStatistics = gql` ` export const searchAdminUsers = gql` - query { - searchAdminUsers { + query ($pageSize: Int = 25, $currentPage: Int = 1, $order: Order = ASC) { + searchAdminUsers(pageSize: $pageSize, currentPage: $currentPage, order: $order) { userCount userList { firstName diff --git a/frontend/src/pages/InfoStatistic.vue b/frontend/src/pages/InfoStatistic.vue index d5cebf2b1..b63a09423 100644 --- a/frontend/src/pages/InfoStatistic.vue +++ b/frontend/src/pages/InfoStatistic.vue @@ -49,7 +49,11 @@ const moderators = computed(() => itemsAdminUser.value.filter((item) => item.rol const { onResult: onContributionLinksResult, onError: onContributionLinksError } = useQuery(listContributionLinks) -const { onResult: onAdminUsersResult, onError: onAdminUsersError } = useQuery(searchAdminUsers) +const { onResult: onAdminUsersResult, onError: onAdminUsersError } = useQuery(searchAdminUsers, { + pageSize: 25, + currentPage: 1, + order: 'ASC', +}) onContributionLinksResult(({ data }) => { if (data) { From 4b5be2454feebf58b83d13dd1a026e1474eea5c1 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:07:00 +0200 Subject: [PATCH 07/21] refactor database to speed up --- database/package.json | 13 ++- database/src/clear.ts | 55 +++++++++++ database/src/config/index.ts | 4 +- database/src/index.ts | 54 ++++++++++- database/src/prepare.ts | 80 ++++++++++------ database/yarn.lock | 178 +++++++++++++++++++++++++++++++++++ 6 files changed, 348 insertions(+), 36 deletions(-) create mode 100644 database/src/clear.ts diff --git a/database/package.json b/database/package.json index 47af20975..ed4115b78 100644 --- a/database/package.json +++ b/database/package.json @@ -13,10 +13,14 @@ "up": "cross-env TZ=UTC node build/src/index.js up", "down": "cross-env TZ=UTC node build/src/index.js down", "reset": "cross-env TZ=UTC node build/src/index.js reset", - "dev_up": "cross-env TZ=UTC ts-node src/index.ts up", - "dev_down": "cross-env TZ=UTC ts-node src/index.ts down", - "dev_reset": "cross-env TZ=UTC ts-node src/index.ts reset", - "lint": "eslint --max-warnings=0 --ext .js,.ts ." + "clear": "cross-env TZ=UTC tsx src/index.ts clear", + "dev_up": "cross-env TZ=UTC tsx src/index.ts up", + "dev_down": "cross-env TZ=UTC tsx src/index.ts down", + "dev_reset": "cross-env TZ=UTC tsx src/index.ts reset", + "lint": "eslint --max-warnings=0 --ext .js,.ts .", + "up:backend_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_backend tsx src/index.ts up", + "up:federation_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_federation tsx src/index.ts up", + "up:dht_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_dht tsx src/index.ts up" }, "devDependencies": { "@eslint-community/eslint-plugin-eslint-comments": "^3.2.1", @@ -38,6 +42,7 @@ "ncp": "^2.0.0", "prettier": "^2.8.7", "ts-node": "^10.9.2", + "tsx": "^4.19.3", "typescript": "^4.9.5" }, "dependencies": { diff --git a/database/src/clear.ts b/database/src/clear.ts new file mode 100644 index 000000000..67b456c98 --- /dev/null +++ b/database/src/clear.ts @@ -0,0 +1,55 @@ +/* eslint-disable no-console */ +import { Connection, createConnection } from 'mysql2/promise' +import { CONFIG } from './config' + +export async function truncateTables(connection: Connection) { + const [tables] = await connection.query('SHOW TABLES') + const tableNames = (tables as any[]).map((table) => Object.values(table)[0]) + + if (tableNames.length === 0) { + // No tables found in database. + return + } + + // Disabling foreign key checks... + await connection.query('SET FOREIGN_KEY_CHECKS = 0') + + // Truncating all tables... + for (const tableName of tableNames) { + if (tableName === CONFIG.MIGRATIONS_TABLE) { + continue + } + await connection.query(`TRUNCATE TABLE \`${tableName}\``) + } + + // Re-enabling foreign key checks... + await connection.query('SET FOREIGN_KEY_CHECKS = 1') +} + +export async function clearDatabase() { + const connection = await createConnection({ + host: CONFIG.DB_HOST, + port: CONFIG.DB_PORT, + user: CONFIG.DB_USER, + password: CONFIG.DB_PASSWORD, + database: CONFIG.DB_DATABASE, + }) + + await truncateTables(connection) + + // Database cleared successfully. + await connection.end() +} + +// Execute if this file is run directly +if (require.main === module) { + clearDatabase() + .then(() => { + // Database clear operation completed. + process.exit(0) + }) + .catch((error) => { + console.error('Failed to clear database:', error) + process.exit(1) + }) +} diff --git a/database/src/config/index.ts b/database/src/config/index.ts index 0fc8cf9c5..1a5fb08ed 100644 --- a/database/src/config/index.ts +++ b/database/src/config/index.ts @@ -24,6 +24,8 @@ const migrations = { MIGRATIONS_TABLE: process.env.MIGRATIONS_TABLE || 'migrations', } +const nodeEnv = process.env.NODE_ENV || 'development' + // Check config version constants.CONFIG_VERSION.CURRENT = process.env.CONFIG_VERSION || constants.CONFIG_VERSION.DEFAULT if ( @@ -36,4 +38,4 @@ if ( ) } -export const CONFIG = { ...constants, ...database, ...migrations } +export const CONFIG = { ...constants, ...database, ...migrations, NODE_ENV: nodeEnv } diff --git a/database/src/index.ts b/database/src/index.ts index 96785a721..86ac05c79 100644 --- a/database/src/index.ts +++ b/database/src/index.ts @@ -1,14 +1,43 @@ -import { createDatabase } from './prepare' +import { DatabaseState, getDatabaseState } from './prepare' import { CONFIG } from './config' import { createPool } from 'mysql' import { Migration } from 'ts-mysql-migrate' +import { clearDatabase } from './clear' +import { latestDbVersion } from './config/detectLastDBVersion' import path from 'path' const run = async (command: string) => { + if (command === 'clear') { + if (CONFIG.NODE_ENV === 'production') { + throw new Error('Clearing database in production is not allowed') + } + await clearDatabase() + return + } // Database actions not supported by our migration library - await createDatabase() - + // await createDatabase() + const state = await getDatabaseState() + if (state === DatabaseState.NOT_CONNECTED) { + throw new Error( + `Database not connected, is database server running? + host: ${CONFIG.DB_HOST} + port: ${CONFIG.DB_PORT} + user: ${CONFIG.DB_USER} + password: ${CONFIG.DB_PASSWORD.slice(-2)} + database: ${CONFIG.DB_DATABASE}`, + ) + } + if (state === DatabaseState.HIGHER_VERSION) { + throw new Error('Database version is higher than required, please switch to the correct branch') + } + if (state === DatabaseState.SAME_VERSION) { + if (command === 'up') { + // eslint-disable-next-line no-console + console.log('Database is up to date') + return + } + } // Initialize Migrations const pool = createPool({ host: CONFIG.DB_HOST, @@ -34,12 +63,29 @@ const run = async (command: string) => { await migration.down() // use for downgrade script break case 'reset': - // TODO protect from production + if (CONFIG.NODE_ENV === 'production') { + throw new Error('Resetting database in production is not allowed') + } await migration.reset() break default: throw new Error(`Unsupported command ${command}`) } + if (command === 'reset') { + // eslint-disable-next-line no-console + console.log('Database was reset') + } else { + const currentDbVersion = await migration.getLastVersion() + // eslint-disable-next-line no-console + console.log(`Database was ${command} migrated to version: ${currentDbVersion.fileName}`) + if (latestDbVersion === currentDbVersion.fileName.split('.')[0]) { + // eslint-disable-next-line no-console + console.log('Database is now up to date') + } else { + // eslint-disable-next-line no-console + console.log('The latest database version is: ', latestDbVersion) + } + } // Terminate connections gracefully pool.end() diff --git a/database/src/prepare.ts b/database/src/prepare.ts index 2ecbb4c42..ee60fc496 100644 --- a/database/src/prepare.ts +++ b/database/src/prepare.ts @@ -1,38 +1,64 @@ -import { createConnection } from 'mysql2/promise' +/* eslint-disable no-unused-vars */ +import { Connection, createConnection, ResultSetHeader, RowDataPacket } from 'mysql2/promise' import { CONFIG } from './config' +import { latestDbVersion } from './config/detectLastDBVersion' -export const createDatabase = async (): Promise => { - const con = await createConnection({ - host: CONFIG.DB_HOST, - port: CONFIG.DB_PORT, - user: CONFIG.DB_USER, - password: CONFIG.DB_PASSWORD, - }) +export enum DatabaseState { + NOT_CONNECTED = 'NOT_CONNECTED', + LOWER_VERSION = 'LOWER_VERSION', + HIGHER_VERSION = 'HIGHER_VERSION', + SAME_VERSION = 'SAME_VERSION', +} - await con.connect() +async function connectToDatabaseServer(): Promise { + try { + return await createConnection({ + host: CONFIG.DB_HOST, + port: CONFIG.DB_PORT, + user: CONFIG.DB_USER, + password: CONFIG.DB_PASSWORD, + }) + } catch (error) { + return null + } +} - // Create Database `gradido_community` - await con.query(` - CREATE DATABASE IF NOT EXISTS ${CONFIG.DB_DATABASE} +export const getDatabaseState = async (): Promise => { + const connection = await connectToDatabaseServer() + if (!connection) { + return DatabaseState.NOT_CONNECTED + } + + // make sure the database exists + const [result] = await connection.query(` + CREATE DATABASE IF NOT EXISTS ${CONFIG.DB_DATABASE} DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;`) - /* LEGACY CODE - import { RowDataPacket } from 'mysql2/promise' - // Check if old migration table is present, delete if needed - const [rows] = await con.query(`SHOW TABLES FROM \`${CONFIG.DB_DATABASE}\` LIKE 'migrations';`) - if ((rows).length > 0) { - const [rows] = await con.query( - `SHOW COLUMNS FROM \`${CONFIG.DB_DATABASE}\`.\`migrations\` LIKE 'db_version';`, - ) - if ((rows).length > 0) { - await con.query(`DROP TABLE \`${CONFIG.DB_DATABASE}\`.\`migrations\``) - // eslint-disable-next-line no-console - console.log('Found and dropped old migrations table') - } + if (result.affectedRows === 1) { + // eslint-disable-next-line no-console + console.log(`Database ${CONFIG.DB_DATABASE} created`) + return DatabaseState.LOWER_VERSION } - */ - await con.end() + await connection.query(`USE ${CONFIG.DB_DATABASE}`) + try { + // check if the database is up to date + const [rows] = await connection.query( + `SELECT * FROM ${CONFIG.MIGRATIONS_TABLE} ORDER BY version DESC LIMIT 1`, + ) + if (rows.length === 0) { + return DatabaseState.LOWER_VERSION + } + connection.destroy() + const dbVersion = rows[0].fileName.split('.')[0] + return dbVersion === latestDbVersion + ? DatabaseState.SAME_VERSION + : dbVersion < latestDbVersion + ? DatabaseState.LOWER_VERSION + : DatabaseState.HIGHER_VERSION + } catch (error) { + return DatabaseState.LOWER_VERSION + } } diff --git a/database/yarn.lock b/database/yarn.lock index 491470016..c07fbe5a1 100644 --- a/database/yarn.lock +++ b/database/yarn.lock @@ -16,6 +16,131 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" +"@esbuild/aix-ppc64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.25.3.tgz#014180d9a149cffd95aaeead37179433f5ea5437" + integrity sha512-W8bFfPA8DowP8l//sxjJLSLkD8iEjMc7cBVyP+u4cEv9sM7mdUCkgsj+t0n/BWPFtv7WWCN5Yzj0N6FJNUUqBQ== + +"@esbuild/android-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.25.3.tgz#649e47e04ddb24a27dc05c395724bc5f4c55cbfe" + integrity sha512-XelR6MzjlZuBM4f5z2IQHK6LkK34Cvv6Rj2EntER3lwCBFdg6h2lKbtRjpTTsdEjD/WSe1q8UyPBXP1x3i/wYQ== + +"@esbuild/android-arm@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.25.3.tgz#8a0f719c8dc28a4a6567ef7328c36ea85f568ff4" + integrity sha512-PuwVXbnP87Tcff5I9ngV0lmiSu40xw1At6i3GsU77U7cjDDB4s0X2cyFuBiDa1SBk9DnvWwnGvVaGBqoFWPb7A== + +"@esbuild/android-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.25.3.tgz#e2ab182d1fd06da9bef0784a13c28a7602d78009" + integrity sha512-ogtTpYHT/g1GWS/zKM0cc/tIebFjm1F9Aw1boQ2Y0eUQ+J89d0jFY//s9ei9jVIlkYi8AfOjiixcLJSGNSOAdQ== + +"@esbuild/darwin-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.25.3.tgz#c7f3166fcece4d158a73dcfe71b2672ca0b1668b" + integrity sha512-eESK5yfPNTqpAmDfFWNsOhmIOaQA59tAcF/EfYvo5/QWQCzXn5iUSOnqt3ra3UdzBv073ykTtmeLJZGt3HhA+w== + +"@esbuild/darwin-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.25.3.tgz#d8c5342ec1a4bf4b1915643dfe031ba4b173a87a" + integrity sha512-Kd8glo7sIZtwOLcPbW0yLpKmBNWMANZhrC1r6K++uDR2zyzb6AeOYtI6udbtabmQpFaxJ8uduXMAo1gs5ozz8A== + +"@esbuild/freebsd-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.3.tgz#9f7d789e2eb7747d4868817417cc968ffa84f35b" + integrity sha512-EJiyS70BYybOBpJth3M0KLOus0n+RRMKTYzhYhFeMwp7e/RaajXvP+BWlmEXNk6uk+KAu46j/kaQzr6au+JcIw== + +"@esbuild/freebsd-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.25.3.tgz#8ad35c51d084184a8e9e76bb4356e95350a64709" + integrity sha512-Q+wSjaLpGxYf7zC0kL0nDlhsfuFkoN+EXrx2KSB33RhinWzejOd6AvgmP5JbkgXKmjhmpfgKZq24pneodYqE8Q== + +"@esbuild/linux-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.25.3.tgz#3af0da3d9186092a9edd4e28fa342f57d9e3cd30" + integrity sha512-xCUgnNYhRD5bb1C1nqrDV1PfkwgbswTTBRbAd8aH5PhYzikdf/ddtsYyMXFfGSsb/6t6QaPSzxtbfAZr9uox4A== + +"@esbuild/linux-arm@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.25.3.tgz#e91cafa95e4474b3ae3d54da12e006b782e57225" + integrity sha512-dUOVmAUzuHy2ZOKIHIKHCm58HKzFqd+puLaS424h6I85GlSDRZIA5ycBixb3mFgM0Jdh+ZOSB6KptX30DD8YOQ== + +"@esbuild/linux-ia32@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.25.3.tgz#81025732d85b68ee510161b94acdf7e3007ea177" + integrity sha512-yplPOpczHOO4jTYKmuYuANI3WhvIPSVANGcNUeMlxH4twz/TeXuzEP41tGKNGWJjuMhotpGabeFYGAOU2ummBw== + +"@esbuild/linux-loong64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.25.3.tgz#3c744e4c8d5e1148cbe60a71a11b58ed8ee5deb8" + integrity sha512-P4BLP5/fjyihmXCELRGrLd793q/lBtKMQl8ARGpDxgzgIKJDRJ/u4r1A/HgpBpKpKZelGct2PGI4T+axcedf6g== + +"@esbuild/linux-mips64el@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.25.3.tgz#1dfe2a5d63702db9034cc6b10b3087cc0424ec26" + integrity sha512-eRAOV2ODpu6P5divMEMa26RRqb2yUoYsuQQOuFUexUoQndm4MdpXXDBbUoKIc0iPa4aCO7gIhtnYomkn2x+bag== + +"@esbuild/linux-ppc64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.25.3.tgz#2e85d9764c04a1ebb346dc0813ea05952c9a5c56" + integrity sha512-ZC4jV2p7VbzTlnl8nZKLcBkfzIf4Yad1SJM4ZMKYnJqZFD4rTI+pBG65u8ev4jk3/MPwY9DvGn50wi3uhdaghg== + +"@esbuild/linux-riscv64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.25.3.tgz#a9ea3334556b09f85ccbfead58c803d305092415" + integrity sha512-LDDODcFzNtECTrUUbVCs6j9/bDVqy7DDRsuIXJg6so+mFksgwG7ZVnTruYi5V+z3eE5y+BJZw7VvUadkbfg7QA== + +"@esbuild/linux-s390x@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.25.3.tgz#f6a7cb67969222b200974de58f105dfe8e99448d" + integrity sha512-s+w/NOY2k0yC2p9SLen+ymflgcpRkvwwa02fqmAwhBRI3SC12uiS10edHHXlVWwfAagYSY5UpmT/zISXPMW3tQ== + +"@esbuild/linux-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.25.3.tgz#a237d3578ecdd184a3066b1f425e314ade0f8033" + integrity sha512-nQHDz4pXjSDC6UfOE1Fw9Q8d6GCAd9KdvMZpfVGWSJztYCarRgSDfOVBY5xwhQXseiyxapkiSJi/5/ja8mRFFA== + +"@esbuild/netbsd-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.3.tgz#4c15c68d8149614ddb6a56f9c85ae62ccca08259" + integrity sha512-1QaLtOWq0mzK6tzzp0jRN3eccmN3hezey7mhLnzC6oNlJoUJz4nym5ZD7mDnS/LZQgkrhEbEiTn515lPeLpgWA== + +"@esbuild/netbsd-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.25.3.tgz#12f6856f8c54c2d7d0a8a64a9711c01a743878d5" + integrity sha512-i5Hm68HXHdgv8wkrt+10Bc50zM0/eonPb/a/OFVfB6Qvpiirco5gBA5bz7S2SHuU+Y4LWn/zehzNX14Sp4r27g== + +"@esbuild/openbsd-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.3.tgz#ca078dad4a34df192c60233b058db2ca3d94bc5c" + integrity sha512-zGAVApJEYTbOC6H/3QBr2mq3upG/LBEXr85/pTtKiv2IXcgKV0RT0QA/hSXZqSvLEpXeIxah7LczB4lkiYhTAQ== + +"@esbuild/openbsd-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.25.3.tgz#c9178adb60e140e03a881d0791248489c79f95b2" + integrity sha512-fpqctI45NnCIDKBH5AXQBsD0NDPbEFczK98hk/aa6HJxbl+UtLkJV2+Bvy5hLSLk3LHmqt0NTkKNso1A9y1a4w== + +"@esbuild/sunos-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.25.3.tgz#03765eb6d4214ff27e5230af779e80790d1ee09f" + integrity sha512-ROJhm7d8bk9dMCUZjkS8fgzsPAZEjtRJqCAmVgB0gMrvG7hfmPmz9k1rwO4jSiblFjYmNvbECL9uhaPzONMfgA== + +"@esbuild/win32-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.25.3.tgz#f1c867bd1730a9b8dfc461785ec6462e349411ea" + integrity sha512-YWcow8peiHpNBiIXHwaswPnAXLsLVygFwCB3A7Bh5jRkIBFWHGmNQ48AlX4xDvQNoMZlPYzjVOQDYEzWCqufMQ== + +"@esbuild/win32-ia32@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.25.3.tgz#77491f59ef6c9ddf41df70670d5678beb3acc322" + integrity sha512-qspTZOIGoXVS4DpNqUYUs9UxVb04khS1Degaw/MnfMe7goQ3lTfQ13Vw4qY/Nj0979BGvMRpAYbs/BAxEvU8ew== + +"@esbuild/win32-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.25.3.tgz#b17a2171f9074df9e91bfb07ef99a892ac06412a" + integrity sha512-ICgUR+kPimx0vvRzf+N/7L7tVSQeE3BYY+NhHRHXS1kBuPO7z2+7ea2HbhDyZdTephgvNvKrlDDKUexuCVBVvg== + "@eslint-community/eslint-plugin-eslint-comments@^3.2.1": version "3.2.1" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-plugin-eslint-comments/-/eslint-plugin-eslint-comments-3.2.1.tgz#3c65061e27f155eae3744c3b30c5a8253a959040" @@ -763,6 +888,37 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" +esbuild@~0.25.0: + version "0.25.3" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.25.3.tgz#371f7cb41283e5b2191a96047a7a89562965a285" + integrity sha512-qKA6Pvai73+M2FtftpNKRxJ78GIjmFXFxd/1DVBqGo/qNhLSfv+G12n9pNoWdytJC8U00TrViOwpjT0zgqQS8Q== + optionalDependencies: + "@esbuild/aix-ppc64" "0.25.3" + "@esbuild/android-arm" "0.25.3" + "@esbuild/android-arm64" "0.25.3" + "@esbuild/android-x64" "0.25.3" + "@esbuild/darwin-arm64" "0.25.3" + "@esbuild/darwin-x64" "0.25.3" + "@esbuild/freebsd-arm64" "0.25.3" + "@esbuild/freebsd-x64" "0.25.3" + "@esbuild/linux-arm" "0.25.3" + "@esbuild/linux-arm64" "0.25.3" + "@esbuild/linux-ia32" "0.25.3" + "@esbuild/linux-loong64" "0.25.3" + "@esbuild/linux-mips64el" "0.25.3" + "@esbuild/linux-ppc64" "0.25.3" + "@esbuild/linux-riscv64" "0.25.3" + "@esbuild/linux-s390x" "0.25.3" + "@esbuild/linux-x64" "0.25.3" + "@esbuild/netbsd-arm64" "0.25.3" + "@esbuild/netbsd-x64" "0.25.3" + "@esbuild/openbsd-arm64" "0.25.3" + "@esbuild/openbsd-x64" "0.25.3" + "@esbuild/sunos-x64" "0.25.3" + "@esbuild/win32-arm64" "0.25.3" + "@esbuild/win32-ia32" "0.25.3" + "@esbuild/win32-x64" "0.25.3" + escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -1123,6 +1279,11 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= +fsevents@~2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -1199,6 +1360,13 @@ get-tsconfig@^4.5.0: dependencies: resolve-pkg-maps "^1.0.0" +get-tsconfig@^4.7.5: + version "4.10.0" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.10.0.tgz#403a682b373a823612475a4c2928c7326fc0f6bb" + integrity sha512-kGzZ3LWWQcGIAmg6iWvXn0ei6WDtV26wzHRMwDSzmAbcXrTEXxHy6IehI6/4eT6VRKyMP1eF1VqwrVUmE/LR7A== + dependencies: + resolve-pkg-maps "^1.0.0" + glob-parent@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -2412,6 +2580,16 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tsx@^4.19.3: + version "4.19.3" + resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.19.3.tgz#2bdbcb87089374d933596f8645615142ed727666" + integrity sha512-4H8vUNGNjQ4V2EOoGw005+c+dGuPSnhpPBPHBtsZdGZBk/iJb4kguGlPWaZTZ3q5nMtFOEsY0nRDlh9PJyd6SQ== + dependencies: + esbuild "~0.25.0" + get-tsconfig "^4.7.5" + optionalDependencies: + fsevents "~2.3.3" + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" From abc56742ae48b0b1605367738148fbe6d11d6c4d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:07:27 +0200 Subject: [PATCH 08/21] use specific test db for module --- backend/package.json | 11 ++++++----- dht-node/package.json | 2 +- federation/package.json | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/package.json b/backend/package.json index 880d48a53..83175e0e3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -13,11 +13,11 @@ "start": "cross-env TZ=UTC TS_NODE_BASEURL=./build node -r tsconfig-paths/register build/src/index.js", "dev": "cross-env TZ=UTC nodemon -w src --ext ts,pug,json,css --exec ts-node -r tsconfig-paths/register src/index.ts", "lint": "eslint --max-warnings=0 .", - "test": "cross-env TZ=UTC NODE_ENV=development jest --runInBand --forceExit --detectOpenHandles", - "seed": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/seeds/index.ts", - "klicktipp": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/util/executeKlicktipp.ts", - "gmsusers": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/apis/gms/ExportUsers.ts", - "humhubUserExport": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/apis/humhub/ExportUsers.ts", + "test": "cross-env TZ=UTC NODE_ENV=development DB_DATABASE=gradido_test_backend jest --runInBand --forceExit --detectOpenHandles", + "seed": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/seeds/index.ts", + "klicktipp": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/util/executeKlicktipp.ts", + "gmsusers": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/apis/gms/ExportUsers.ts", + "humhubUserExport": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/apis/humhub/ExportUsers.ts", "locales": "scripts/sort.sh" }, "dependencies": { @@ -94,6 +94,7 @@ "prettier": "^2.8.7", "ts-jest": "^27.0.5", "ts-node": "^10.9.2", + "tsx": "^4.19.3", "tsconfig-paths": "^3.14.0", "typescript": "^4.9.5" }, diff --git a/dht-node/package.json b/dht-node/package.json index aa86b5b2f..ed3245793 100644 --- a/dht-node/package.json +++ b/dht-node/package.json @@ -13,7 +13,7 @@ "start": "cross-env TZ=UTC TS_NODE_BASEURL=./build node -r tsconfig-paths/register build/src/index.js", "dev": "cross-env TZ=UTC nodemon -w src --ext ts --exec ts-node -r dotenv/config -r tsconfig-paths/register src/index.ts", "lint": "eslint --max-warnings=0 --ext .js,.ts .", - "test": "cross-env TZ=UTC NODE_ENV=development jest --runInBand --forceExit --detectOpenHandles" + "test": "cross-env TZ=UTC NODE_ENV=development DB_DATABASE=gradido_test_dht jest --runInBand --forceExit --detectOpenHandles" }, "dependencies": { "@hyperswarm/dht": "^6.4.4", diff --git a/federation/package.json b/federation/package.json index d54562abd..d76696ddb 100644 --- a/federation/package.json +++ b/federation/package.json @@ -11,7 +11,7 @@ "build": "tsc --build", "clean": "tsc --build --clean", "start": "cross-env TZ=UTC TS_NODE_BASEURL=./build node -r tsconfig-paths/register build/src/index.js", - "test": "cross-env TZ=UTC NODE_ENV=development jest --runInBand --forceExit --detectOpenHandles", + "test": "cross-env TZ=UTC NODE_ENV=development DB_DATABASE=gradido_test_federation jest --runInBand --forceExit --detectOpenHandles", "dev": "cross-env TZ=UTC nodemon -w src --ext ts --exec ts-node -r dotenv/config -r tsconfig-paths/register src/index.ts", "lint": "eslint --max-warnings=0 --ext .js,.ts ." }, From 169735b77470bf40ee480935f5a58a14b6d893e4 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:08:10 +0200 Subject: [PATCH 09/21] optimize github workflows, use specific test db --- .github/workflows/test_backend.yml | 20 ++++++------ .github/workflows/test_database.yml | 2 +- .github/workflows/test_dht_node.yml | 44 ++++++++------------------- .github/workflows/test_e2e.yml | 38 +++++++++++------------ .github/workflows/test_federation.yml | 23 ++++++-------- 5 files changed, 52 insertions(+), 75 deletions(-) diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index 8d1a8e2c4..4461940b3 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -43,18 +43,20 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Backend | docker-compose mariadb + - name: docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps mariadb + + - name: Backend | install and build + run: cd database && yarn && yarn build && cd ../config && yarn && cd ../backend && yarn && yarn build - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash + - name: wait for database to be ready + run: docker run --rm --network gradido_internal-net busybox sh -c 'until nc -z mariadb 3306; do echo waiting for db; sleep 1; done;' - - name: Backend | docker-compose database - run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps database + - name: Backend | prepare database + run: cd database && yarn up:backend_test - - name: Backend | Unit tests - run: cd database && yarn && yarn build && cd ../config && yarn install && cd ../backend && yarn && yarn test + - name: Backend | Unit tests + run: cd backend && yarn test lint: if: needs.files-changed.outputs.backend == 'true' @@ -66,7 +68,7 @@ jobs: uses: actions/checkout@v3 - name: Backend | Lint - run: cd database && yarn && cd ../config && yarn install && cd ../backend && yarn && yarn run lint + run: cd backend && yarn && yarn run lint locales: if: needs.files-changed.outputs.backend == 'true' diff --git a/.github/workflows/test_database.yml b/.github/workflows/test_database.yml index 6f0c1d760..aa02edf87 100644 --- a/.github/workflows/test_database.yml +++ b/.github/workflows/test_database.yml @@ -42,7 +42,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Database | docker-compose + - name: docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach mariadb - name: Database | up diff --git a/.github/workflows/test_dht_node.yml b/.github/workflows/test_dht_node.yml index 4cb50a093..0fed7b2ea 100644 --- a/.github/workflows/test_dht_node.yml +++ b/.github/workflows/test_dht_node.yml @@ -31,15 +31,7 @@ jobs: uses: actions/checkout@v3 - name: Build 'test' image - run: | - docker build --target test -t "gradido/dht-node:test" -f dht-node/Dockerfile . - docker save "gradido/dht-node:test" > /tmp/dht-node.tar - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: docker-dht-node-test - path: /tmp/dht-node.tar + run: docker build --target test -t "gradido/dht-node:test" -f dht-node/Dockerfile . lint: name: Lint - DHT Node @@ -50,8 +42,8 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Lint - run: cd database && yarn && cd ../config && yarn install && cd ../dht-node && yarn && yarn run lint + - name: DHT-Node | Lint + run: cd dht-node && yarn && yarn run lint unit_test: name: Unit Tests - DHT Node @@ -62,30 +54,18 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Download Docker Image - uses: actions/download-artifact@v4 - with: - name: docker-dht-node-test - path: /tmp - - - name: Load Docker Image - run: docker load < /tmp/dht-node.tar - - name: docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps mariadb - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash + - name: DHT-Node | install and build + run: cd database && yarn && yarn build && cd ../config && yarn && cd ../dht-node && yarn && yarn build - - name: docker-compose database - run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps database + - name: wait for database to be ready + run: docker run --rm --network gradido_internal-net busybox sh -c 'until nc -z mariadb 3306; do echo waiting for db; sleep 1; done;' - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash + - name: DHT-Node | prepare database + run: cd database && yarn up:dht_test + + - name: DHT-Node | Unit tests + run: cd dht-node && yarn test - - name: Unit tests - run: cd database && yarn && yarn build && cd ../config && yarn install && cd ../dht-node && yarn && yarn test - #- name: Unit tests - # run: docker run --env NODE_ENV=test --env DB_HOST=mariadb --network gradido_internal-net --rm gradido/dht-node:test yarn run test diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml index 3fb92562c..6dbc31704 100644 --- a/.github/workflows/test_e2e.yml +++ b/.github/workflows/test_e2e.yml @@ -13,25 +13,10 @@ jobs: - name: Boot up test system | docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach mariadb - - name: Sleep for 10 seconds - run: sleep 10s - - - name: Boot up test system | seed backend - run: | - sudo chown runner:docker -R * - cd database - yarn && yarn dev_reset - cd ../config - yarn install - cd ../backend - yarn && yarn seed - - - name: Boot up test system | docker-compose backend, frontend, admin, nginx, mailserver - run: | - cd backend - cp .env.test_e2e .env - cd .. - docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps backend frontend admin nginx mailserver + - name: Prepare test system + run: sudo chown runner:docker -R * + cd database && yarn + cd ../backend && yarn - name: End-to-end tests | prepare run: | @@ -41,6 +26,21 @@ jobs: cd e2e-tests/ yarn + - name: wait for database to be ready + run: docker run --rm --network gradido_internal-net busybox sh -c 'until nc -z mariadb 3306; do echo waiting for db; sleep 1; done;' + + - name: Boot up test system | seed backend + run: | + cd database && yarn dev_reset + cd ../backend && yarn seed + + - name: Boot up test system | docker-compose backend, frontend, admin, nginx, mailserver + run: | + cd backend + cp .env.test_e2e .env + cd .. + docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps backend frontend admin nginx mailserver + - name: End-to-end tests | run tests id: e2e-tests run: | diff --git a/.github/workflows/test_federation.yml b/.github/workflows/test_federation.yml index 8f5acb344..8e912d41a 100644 --- a/.github/workflows/test_federation.yml +++ b/.github/workflows/test_federation.yml @@ -72,20 +72,15 @@ jobs: - name: docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps mariadb - - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash - - name: docker-compose database - run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps database + - name: Federation | install and build + run: cd database && yarn && yarn build && cd ../config && yarn && cd ../federation && yarn && yarn build - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash + - name: wait for database to be ready + run: docker run --rm --network gradido_internal-net busybox sh -c 'until nc -z mariadb 3306; do echo waiting for db; sleep 1; done;' - #- name: Unit tests - # run: cd database && yarn && yarn build && cd ../dht-node && yarn && yarn test - - name: Unit tests - run: | - docker run --env NODE_ENV=test --env DB_HOST=mariadb --network gradido_internal-net --rm gradido/federation:test yarn run test + - name: Federation | prepare database + run: cd database && yarn up:federation_test + + - name: Federation | Unit tests + run: docker run --env NODE_ENV=test --env DB_HOST=mariadb --network gradido_internal-net --rm gradido/federation:test yarn run test From 73d688eeb340cd8aef6fad91a2b9fcf3c3b9738d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:16:02 +0200 Subject: [PATCH 10/21] update yarn.lock --- backend/yarn.lock | 170 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 2 deletions(-) diff --git a/backend/yarn.lock b/backend/yarn.lock index f7a2a27b1..1bc87a194 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -347,6 +347,131 @@ dependencies: tslib "^2.4.0" +"@esbuild/aix-ppc64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.25.3.tgz#014180d9a149cffd95aaeead37179433f5ea5437" + integrity sha512-W8bFfPA8DowP8l//sxjJLSLkD8iEjMc7cBVyP+u4cEv9sM7mdUCkgsj+t0n/BWPFtv7WWCN5Yzj0N6FJNUUqBQ== + +"@esbuild/android-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.25.3.tgz#649e47e04ddb24a27dc05c395724bc5f4c55cbfe" + integrity sha512-XelR6MzjlZuBM4f5z2IQHK6LkK34Cvv6Rj2EntER3lwCBFdg6h2lKbtRjpTTsdEjD/WSe1q8UyPBXP1x3i/wYQ== + +"@esbuild/android-arm@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.25.3.tgz#8a0f719c8dc28a4a6567ef7328c36ea85f568ff4" + integrity sha512-PuwVXbnP87Tcff5I9ngV0lmiSu40xw1At6i3GsU77U7cjDDB4s0X2cyFuBiDa1SBk9DnvWwnGvVaGBqoFWPb7A== + +"@esbuild/android-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.25.3.tgz#e2ab182d1fd06da9bef0784a13c28a7602d78009" + integrity sha512-ogtTpYHT/g1GWS/zKM0cc/tIebFjm1F9Aw1boQ2Y0eUQ+J89d0jFY//s9ei9jVIlkYi8AfOjiixcLJSGNSOAdQ== + +"@esbuild/darwin-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.25.3.tgz#c7f3166fcece4d158a73dcfe71b2672ca0b1668b" + integrity sha512-eESK5yfPNTqpAmDfFWNsOhmIOaQA59tAcF/EfYvo5/QWQCzXn5iUSOnqt3ra3UdzBv073ykTtmeLJZGt3HhA+w== + +"@esbuild/darwin-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.25.3.tgz#d8c5342ec1a4bf4b1915643dfe031ba4b173a87a" + integrity sha512-Kd8glo7sIZtwOLcPbW0yLpKmBNWMANZhrC1r6K++uDR2zyzb6AeOYtI6udbtabmQpFaxJ8uduXMAo1gs5ozz8A== + +"@esbuild/freebsd-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.3.tgz#9f7d789e2eb7747d4868817417cc968ffa84f35b" + integrity sha512-EJiyS70BYybOBpJth3M0KLOus0n+RRMKTYzhYhFeMwp7e/RaajXvP+BWlmEXNk6uk+KAu46j/kaQzr6au+JcIw== + +"@esbuild/freebsd-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.25.3.tgz#8ad35c51d084184a8e9e76bb4356e95350a64709" + integrity sha512-Q+wSjaLpGxYf7zC0kL0nDlhsfuFkoN+EXrx2KSB33RhinWzejOd6AvgmP5JbkgXKmjhmpfgKZq24pneodYqE8Q== + +"@esbuild/linux-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.25.3.tgz#3af0da3d9186092a9edd4e28fa342f57d9e3cd30" + integrity sha512-xCUgnNYhRD5bb1C1nqrDV1PfkwgbswTTBRbAd8aH5PhYzikdf/ddtsYyMXFfGSsb/6t6QaPSzxtbfAZr9uox4A== + +"@esbuild/linux-arm@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.25.3.tgz#e91cafa95e4474b3ae3d54da12e006b782e57225" + integrity sha512-dUOVmAUzuHy2ZOKIHIKHCm58HKzFqd+puLaS424h6I85GlSDRZIA5ycBixb3mFgM0Jdh+ZOSB6KptX30DD8YOQ== + +"@esbuild/linux-ia32@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.25.3.tgz#81025732d85b68ee510161b94acdf7e3007ea177" + integrity sha512-yplPOpczHOO4jTYKmuYuANI3WhvIPSVANGcNUeMlxH4twz/TeXuzEP41tGKNGWJjuMhotpGabeFYGAOU2ummBw== + +"@esbuild/linux-loong64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.25.3.tgz#3c744e4c8d5e1148cbe60a71a11b58ed8ee5deb8" + integrity sha512-P4BLP5/fjyihmXCELRGrLd793q/lBtKMQl8ARGpDxgzgIKJDRJ/u4r1A/HgpBpKpKZelGct2PGI4T+axcedf6g== + +"@esbuild/linux-mips64el@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.25.3.tgz#1dfe2a5d63702db9034cc6b10b3087cc0424ec26" + integrity sha512-eRAOV2ODpu6P5divMEMa26RRqb2yUoYsuQQOuFUexUoQndm4MdpXXDBbUoKIc0iPa4aCO7gIhtnYomkn2x+bag== + +"@esbuild/linux-ppc64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.25.3.tgz#2e85d9764c04a1ebb346dc0813ea05952c9a5c56" + integrity sha512-ZC4jV2p7VbzTlnl8nZKLcBkfzIf4Yad1SJM4ZMKYnJqZFD4rTI+pBG65u8ev4jk3/MPwY9DvGn50wi3uhdaghg== + +"@esbuild/linux-riscv64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.25.3.tgz#a9ea3334556b09f85ccbfead58c803d305092415" + integrity sha512-LDDODcFzNtECTrUUbVCs6j9/bDVqy7DDRsuIXJg6so+mFksgwG7ZVnTruYi5V+z3eE5y+BJZw7VvUadkbfg7QA== + +"@esbuild/linux-s390x@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.25.3.tgz#f6a7cb67969222b200974de58f105dfe8e99448d" + integrity sha512-s+w/NOY2k0yC2p9SLen+ymflgcpRkvwwa02fqmAwhBRI3SC12uiS10edHHXlVWwfAagYSY5UpmT/zISXPMW3tQ== + +"@esbuild/linux-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.25.3.tgz#a237d3578ecdd184a3066b1f425e314ade0f8033" + integrity sha512-nQHDz4pXjSDC6UfOE1Fw9Q8d6GCAd9KdvMZpfVGWSJztYCarRgSDfOVBY5xwhQXseiyxapkiSJi/5/ja8mRFFA== + +"@esbuild/netbsd-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.3.tgz#4c15c68d8149614ddb6a56f9c85ae62ccca08259" + integrity sha512-1QaLtOWq0mzK6tzzp0jRN3eccmN3hezey7mhLnzC6oNlJoUJz4nym5ZD7mDnS/LZQgkrhEbEiTn515lPeLpgWA== + +"@esbuild/netbsd-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.25.3.tgz#12f6856f8c54c2d7d0a8a64a9711c01a743878d5" + integrity sha512-i5Hm68HXHdgv8wkrt+10Bc50zM0/eonPb/a/OFVfB6Qvpiirco5gBA5bz7S2SHuU+Y4LWn/zehzNX14Sp4r27g== + +"@esbuild/openbsd-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.3.tgz#ca078dad4a34df192c60233b058db2ca3d94bc5c" + integrity sha512-zGAVApJEYTbOC6H/3QBr2mq3upG/LBEXr85/pTtKiv2IXcgKV0RT0QA/hSXZqSvLEpXeIxah7LczB4lkiYhTAQ== + +"@esbuild/openbsd-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.25.3.tgz#c9178adb60e140e03a881d0791248489c79f95b2" + integrity sha512-fpqctI45NnCIDKBH5AXQBsD0NDPbEFczK98hk/aa6HJxbl+UtLkJV2+Bvy5hLSLk3LHmqt0NTkKNso1A9y1a4w== + +"@esbuild/sunos-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.25.3.tgz#03765eb6d4214ff27e5230af779e80790d1ee09f" + integrity sha512-ROJhm7d8bk9dMCUZjkS8fgzsPAZEjtRJqCAmVgB0gMrvG7hfmPmz9k1rwO4jSiblFjYmNvbECL9uhaPzONMfgA== + +"@esbuild/win32-arm64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.25.3.tgz#f1c867bd1730a9b8dfc461785ec6462e349411ea" + integrity sha512-YWcow8peiHpNBiIXHwaswPnAXLsLVygFwCB3A7Bh5jRkIBFWHGmNQ48AlX4xDvQNoMZlPYzjVOQDYEzWCqufMQ== + +"@esbuild/win32-ia32@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.25.3.tgz#77491f59ef6c9ddf41df70670d5678beb3acc322" + integrity sha512-qspTZOIGoXVS4DpNqUYUs9UxVb04khS1Degaw/MnfMe7goQ3lTfQ13Vw4qY/Nj0979BGvMRpAYbs/BAxEvU8ew== + +"@esbuild/win32-x64@0.25.3": + version "0.25.3" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.25.3.tgz#b17a2171f9074df9e91bfb07ef99a892ac06412a" + integrity sha512-ICgUR+kPimx0vvRzf+N/7L7tVSQeE3BYY+NhHRHXS1kBuPO7z2+7ea2HbhDyZdTephgvNvKrlDDKUexuCVBVvg== + "@eslint-community/eslint-plugin-eslint-comments@^3.2.1": version "3.2.1" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-plugin-eslint-comments/-/eslint-plugin-eslint-comments-3.2.1.tgz#3c65061e27f155eae3744c3b30c5a8253a959040" @@ -2986,6 +3111,37 @@ es-to-primitive@^1.3.0: is-date-object "^1.0.5" is-symbol "^1.0.4" +esbuild@~0.25.0: + version "0.25.3" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.25.3.tgz#371f7cb41283e5b2191a96047a7a89562965a285" + integrity sha512-qKA6Pvai73+M2FtftpNKRxJ78GIjmFXFxd/1DVBqGo/qNhLSfv+G12n9pNoWdytJC8U00TrViOwpjT0zgqQS8Q== + optionalDependencies: + "@esbuild/aix-ppc64" "0.25.3" + "@esbuild/android-arm" "0.25.3" + "@esbuild/android-arm64" "0.25.3" + "@esbuild/android-x64" "0.25.3" + "@esbuild/darwin-arm64" "0.25.3" + "@esbuild/darwin-x64" "0.25.3" + "@esbuild/freebsd-arm64" "0.25.3" + "@esbuild/freebsd-x64" "0.25.3" + "@esbuild/linux-arm" "0.25.3" + "@esbuild/linux-arm64" "0.25.3" + "@esbuild/linux-ia32" "0.25.3" + "@esbuild/linux-loong64" "0.25.3" + "@esbuild/linux-mips64el" "0.25.3" + "@esbuild/linux-ppc64" "0.25.3" + "@esbuild/linux-riscv64" "0.25.3" + "@esbuild/linux-s390x" "0.25.3" + "@esbuild/linux-x64" "0.25.3" + "@esbuild/netbsd-arm64" "0.25.3" + "@esbuild/netbsd-x64" "0.25.3" + "@esbuild/openbsd-arm64" "0.25.3" + "@esbuild/openbsd-x64" "0.25.3" + "@esbuild/sunos-x64" "0.25.3" + "@esbuild/win32-arm64" "0.25.3" + "@esbuild/win32-ia32" "0.25.3" + "@esbuild/win32-x64" "0.25.3" + escalade@^3.1.1, escalade@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" @@ -3621,7 +3777,7 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@^2.3.2, fsevents@~2.3.2: +fsevents@^2.3.2, fsevents@~2.3.2, fsevents@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== @@ -3730,7 +3886,7 @@ get-symbol-description@^1.1.0: es-errors "^1.3.0" get-intrinsic "^1.2.6" -get-tsconfig@^4.10.0: +get-tsconfig@^4.10.0, get-tsconfig@^4.7.5: version "4.10.0" resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.10.0.tgz#403a682b373a823612475a4c2928c7326fc0f6bb" integrity sha512-kGzZ3LWWQcGIAmg6iWvXn0ei6WDtV26wzHRMwDSzmAbcXrTEXxHy6IehI6/4eT6VRKyMP1eF1VqwrVUmE/LR7A== @@ -7037,6 +7193,16 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tsx@^4.19.3: + version "4.19.3" + resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.19.3.tgz#2bdbcb87089374d933596f8645615142ed727666" + integrity sha512-4H8vUNGNjQ4V2EOoGw005+c+dGuPSnhpPBPHBtsZdGZBk/iJb4kguGlPWaZTZ3q5nMtFOEsY0nRDlh9PJyd6SQ== + dependencies: + esbuild "~0.25.0" + get-tsconfig "^4.7.5" + optionalDependencies: + fsevents "~2.3.3" + tunnel@0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" From eb57991f03a34d20bb85310ac564ca58706c28b8 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:25:04 +0200 Subject: [PATCH 11/21] adapt to different table name, fix --- .github/workflows/test_dht_node.yml | 2 +- .github/workflows/test_e2e.yml | 3 ++- database/migrations/0044-insert_missing_contributions.ts | 8 ++++---- database/src/index.ts | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_dht_node.yml b/.github/workflows/test_dht_node.yml index 0fed7b2ea..a1aacaf04 100644 --- a/.github/workflows/test_dht_node.yml +++ b/.github/workflows/test_dht_node.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v3 - name: DHT-Node | Lint - run: cd dht-node && yarn && yarn run lint + run: cd database && yarn && cd ../dht-node && yarn && yarn run lint unit_test: name: Unit Tests - DHT Node diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml index 6dbc31704..075373819 100644 --- a/.github/workflows/test_e2e.yml +++ b/.github/workflows/test_e2e.yml @@ -14,7 +14,8 @@ jobs: run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach mariadb - name: Prepare test system - run: sudo chown runner:docker -R * + run: | + sudo chown runner:docker -R * cd database && yarn cd ../backend && yarn diff --git a/database/migrations/0044-insert_missing_contributions.ts b/database/migrations/0044-insert_missing_contributions.ts index a14141498..2435cdd1d 100644 --- a/database/migrations/0044-insert_missing_contributions.ts +++ b/database/migrations/0044-insert_missing_contributions.ts @@ -5,7 +5,7 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { await queryFn( - `INSERT INTO gradido_community.contributions + `INSERT INTO contributions (user_id, created_at, contribution_date, memo, amount, moderator_id, confirmed_by, confirmed_at, transaction_id) SELECT user_id, @@ -18,12 +18,12 @@ SELECT balance_date AS confirmed_at, id FROM - gradido_community.transactions + transactions WHERE type_id = 1 AND NOT EXISTS( - SELECT * FROM gradido_community.contributions - WHERE gradido_community.contributions.transaction_id = gradido_community.transactions.id);`, + SELECT * FROM contributions + WHERE contributions.transaction_id = transactions.id);`, ) } diff --git a/database/src/index.ts b/database/src/index.ts index 86ac05c79..a4034e3d9 100644 --- a/database/src/index.ts +++ b/database/src/index.ts @@ -5,7 +5,7 @@ import { createPool } from 'mysql' import { Migration } from 'ts-mysql-migrate' import { clearDatabase } from './clear' import { latestDbVersion } from './config/detectLastDBVersion' -import path from 'path' +import path from 'node:path' const run = async (command: string) => { if (command === 'clear') { From c849f77851d335f8ec389c9875548faa1bc00b21 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:32:15 +0200 Subject: [PATCH 12/21] try fix workflows --- .github/workflows/test_backend.yml | 2 +- .github/workflows/test_e2e.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index 4461940b3..0a7a5d202 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -68,7 +68,7 @@ jobs: uses: actions/checkout@v3 - name: Backend | Lint - run: cd backend && yarn && yarn run lint + run: cd database && yarn && cd ../backend && yarn && yarn run lint locales: if: needs.files-changed.outputs.backend == 'true' diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml index 075373819..483dc70d4 100644 --- a/.github/workflows/test_e2e.yml +++ b/.github/workflows/test_e2e.yml @@ -16,7 +16,7 @@ jobs: - name: Prepare test system run: | sudo chown runner:docker -R * - cd database && yarn + cd database && yarn && yarn build cd ../backend && yarn - name: End-to-end tests | prepare From 7c06bc82e99bba70e8a69a3c45d76aa2d49b64dc Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:40:47 +0200 Subject: [PATCH 13/21] remove tsx --- backend/package.json | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/package.json b/backend/package.json index 83175e0e3..c0d1ad430 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,10 +14,10 @@ "dev": "cross-env TZ=UTC nodemon -w src --ext ts,pug,json,css --exec ts-node -r tsconfig-paths/register src/index.ts", "lint": "eslint --max-warnings=0 .", "test": "cross-env TZ=UTC NODE_ENV=development DB_DATABASE=gradido_test_backend jest --runInBand --forceExit --detectOpenHandles", - "seed": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/seeds/index.ts", - "klicktipp": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/util/executeKlicktipp.ts", - "gmsusers": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/apis/gms/ExportUsers.ts", - "humhubUserExport": "cross-env TZ=UTC NODE_ENV=development tsx -r tsconfig-paths/register src/apis/humhub/ExportUsers.ts", + "seed": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/seeds/index.ts", + "klicktipp": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/util/executeKlicktipp.ts", + "gmsusers": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/apis/gms/ExportUsers.ts", + "humhubUserExport": "cross-env TZ=UTC NODE_ENV=development ts-node -r tsconfig-paths/register src/apis/humhub/ExportUsers.ts", "locales": "scripts/sort.sh" }, "dependencies": { @@ -94,7 +94,6 @@ "prettier": "^2.8.7", "ts-jest": "^27.0.5", "ts-node": "^10.9.2", - "tsx": "^4.19.3", "tsconfig-paths": "^3.14.0", "typescript": "^4.9.5" }, From 201c6474eb9070da7913bf37d63ef7aa43639502 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 26 Apr 2025 15:44:39 +0200 Subject: [PATCH 14/21] fix workflow --- .github/workflows/test_e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml index 483dc70d4..ae6b4d93e 100644 --- a/.github/workflows/test_e2e.yml +++ b/.github/workflows/test_e2e.yml @@ -17,6 +17,7 @@ jobs: run: | sudo chown runner:docker -R * cd database && yarn && yarn build + cd ../config && yarn cd ../backend && yarn - name: End-to-end tests | prepare From 184a7266e8a44b71f89f75a5e4b3cf0d76cabe64 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 27 Apr 2025 10:41:08 +0200 Subject: [PATCH 15/21] fix warning --- frontend/src/components/Contributions/ContributionList.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/Contributions/ContributionList.vue b/frontend/src/components/Contributions/ContributionList.vue index 889bc8fd2..e7fbd750d 100644 --- a/frontend/src/components/Contributions/ContributionList.vue +++ b/frontend/src/components/Contributions/ContributionList.vue @@ -67,6 +67,7 @@ const props = defineProps({ }) const emit = defineEmits([ + 'close-all-open-collapse', 'update-list-contributions', 'update-contribution-form', 'delete-contribution', From 3478b3b75863b36a929d6392c7776cb6fb439d99 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 27 Apr 2025 10:41:52 +0200 Subject: [PATCH 16/21] fix error with updating wrong contribution list --- frontend/src/pages/Community.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/Community.vue b/frontend/src/pages/Community.vue index 6d7ab8ba0..74cac3210 100644 --- a/frontend/src/pages/Community.vue +++ b/frontend/src/pages/Community.vue @@ -29,7 +29,7 @@ :show-pagination="true" :page-size="pageSize" @close-all-open-collapse="closeAllOpenCollapse" - @update-list-contributions="handleUpdateListAllContributions" + @update-list-contributions="handleUpdateListContributions" @update-contribution-form="handleUpdateContributionForm" @delete-contribution="handleDeleteContribution" @update-status="updateStatus" From f67cee1fe395919e1e1563bbf793a68a42177af2 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 27 Apr 2025 10:42:08 +0200 Subject: [PATCH 17/21] fix bug with not updating contribution message --- .../Contributions/ContributionListItem.vue | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/Contributions/ContributionListItem.vue b/frontend/src/components/Contributions/ContributionListItem.vue index a08af2e4c..2b0e2b0a1 100644 --- a/frontend/src/components/Contributions/ContributionListItem.vue +++ b/frontend/src/components/Contributions/ContributionListItem.vue @@ -246,7 +246,7 @@ function deleteContribution(item) { } } -const { onResult, onError, load } = useLazyQuery(listContributionMessages, { +const { onResult, onError, load, refetch } = useLazyQuery(listContributionMessages, { contributionId: props.contributionId, }) @@ -254,9 +254,15 @@ function getListContributionMessages(closeCollapse = true) { if (closeCollapse) { emit('close-all-open-collapse') } - load(listContributionMessages, { + const variables = { contributionId: props.contributionId, - }) + } + // load works only once and return false on second call + if (!load(listContributionMessages, variables)) { + // update list data every time getListContributionMessages is called + // because it could be added new messages + refetch(variables) + } } onResult((resultValue) => { From 6142ee4ca27c65e646696c6d604d420029f01215 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 27 Apr 2025 10:48:09 +0200 Subject: [PATCH 18/21] fix test --- .../src/components/Contributions/ContributionListItem.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/Contributions/ContributionListItem.spec.js b/frontend/src/components/Contributions/ContributionListItem.spec.js index eb5fb1247..300076060 100644 --- a/frontend/src/components/Contributions/ContributionListItem.spec.js +++ b/frontend/src/components/Contributions/ContributionListItem.spec.js @@ -14,6 +14,7 @@ vi.mock('@vue/apollo-composable', () => ({ onResult: vi.fn(), onError: vi.fn(), load: vi.fn(), + refetch: vi.fn(), })), useMutation: vi.fn(() => ({ mutate: vi.fn(), From 612d8b736b5a9d0918f1bd951b0a81fbf25b35ae Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 27 Apr 2025 15:49:25 +0200 Subject: [PATCH 19/21] show skeleton only as long as transactionslist is loaded --- .../src/components/GddTransactionList.vue | 2 +- frontend/src/graphql/transactions.graphql | 56 ++++++++++ frontend/src/layouts/DashboardLayout.vue | 104 +++++++----------- frontend/src/pages/Community.vue | 4 - frontend/src/pages/InfoStatistic.vue | 8 -- frontend/src/pages/Send.vue | 3 - 6 files changed, 98 insertions(+), 79 deletions(-) create mode 100644 frontend/src/graphql/transactions.graphql diff --git a/frontend/src/components/GddTransactionList.vue b/frontend/src/components/GddTransactionList.vue index 8396f8d52..b4d188374 100644 --- a/frontend/src/components/GddTransactionList.vue +++ b/frontend/src/components/GddTransactionList.vue @@ -101,7 +101,7 @@ export default { this.updateTransactions() }, timestamp: { - immediate: true, + immediate: false, handler: 'updateTransactions', }, }, diff --git a/frontend/src/graphql/transactions.graphql b/frontend/src/graphql/transactions.graphql new file mode 100644 index 000000000..eff0abbb5 --- /dev/null +++ b/frontend/src/graphql/transactions.graphql @@ -0,0 +1,56 @@ +fragment balanceFields on Balance { + balance + balanceGDT + count + linkCount +} + +fragment transactionFields on Transaction { + id + typeId + amount + balance + previousBalance + balanceDate + memo + linkedUser { + firstName + lastName + communityUuid + communityName + gradidoID + alias + } + decay { + decay + start + end + duration + } + linkId +} + +query transactionsQuery($currentPage: Int = 1, $pageSize: Int = 25, $order: Order = DESC) { + transactionList(currentPage: $currentPage, pageSize: $pageSize, order: $order) { + balance { + ...balanceFields + } + transactions { + ...transactionFields + } + } +} + +query transactionsUserCountQuery($currentPage: Int = 1, $pageSize: Int = 25, $order: Order = DESC) { + transactionList(currentPage: $currentPage, pageSize: $pageSize, order: $order) { + balance { + ...balanceFields + } + transactions { + ...transactionFields + } + } + communityStatistics { + totalUsers + } +} \ No newline at end of file diff --git a/frontend/src/layouts/DashboardLayout.vue b/frontend/src/layouts/DashboardLayout.vue index 95bd858f9..6ac0bcb51 100755 --- a/frontend/src/layouts/DashboardLayout.vue +++ b/frontend/src/layouts/DashboardLayout.vue @@ -187,11 +187,10 @@