From 6e63bacba748c41ad91d7199b467483193b08ff4 Mon Sep 17 00:00:00 2001 From: Einhornimmond Date: Fri, 11 Feb 2022 10:15:09 +0100 Subject: [PATCH 1/6] let handle mysql the paging --- backend/src/graphql/resolver/AdminResolver.ts | 16 ++++++---- backend/src/typeorm/repository/User.ts | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/backend/src/graphql/resolver/AdminResolver.ts b/backend/src/graphql/resolver/AdminResolver.ts index 54fe9335e..2b6138dee 100644 --- a/backend/src/graphql/resolver/AdminResolver.ts +++ b/backend/src/graphql/resolver/AdminResolver.ts @@ -32,7 +32,13 @@ export class AdminResolver { @Args() { searchText, currentPage = 1, pageSize = 25, notActivated = false }: SearchUsersArgs, ): Promise { const userRepository = getCustomRepository(UserRepository) - const users = await userRepository.findBySearchCriteria(searchText) + let users: dbUser[] + let count: number + if(notActivated) { + [users, count] = await userRepository.findBySearchCriteriaPagedNotActivated(searchText, currentPage, pageSize) + } else { + [users, count] = await userRepository.findBySearchCriteriaPaged(searchText, currentPage, pageSize) + } let adminUsers = await Promise.all( users.map(async (user) => { const adminUser = new UserAdmin() @@ -41,16 +47,14 @@ export class AdminResolver { adminUser.lastName = user.lastName adminUser.email = user.email adminUser.creation = await getUserCreations(user.id) - adminUser.emailChecked = await hasActivatedEmail(user.email) + adminUser.emailChecked = user.emailChecked adminUser.hasElopage = await hasElopageBuys(user.email) return adminUser }), ) - if (notActivated) adminUsers = adminUsers.filter((u) => !u.emailChecked) - const first = (currentPage - 1) * pageSize return { - userCount: adminUsers.length, - userList: adminUsers.slice(first, first + pageSize), + userCount: count, + userList: adminUsers, } } diff --git a/backend/src/typeorm/repository/User.ts b/backend/src/typeorm/repository/User.ts index 59d6ff465..bf9d55587 100644 --- a/backend/src/typeorm/repository/User.ts +++ b/backend/src/typeorm/repository/User.ts @@ -1,5 +1,6 @@ import { EntityRepository, Repository } from '@dbTools/typeorm' import { User } from '@entity/User' +import internal from 'stream' @EntityRepository(User) export class UserRepository extends Repository { @@ -43,4 +44,34 @@ export class UserRepository extends Repository { ) .getMany() } + async findBySearchCriteriaPaged(searchCriteria: string, currentPage: number, pageSize: number): Promise<[User[], number]> { + return await this.createQueryBuilder('user') + .where( + 'user.firstName like :name or user.lastName like :lastName or user.email like :email', + { + name: `%${searchCriteria}%`, + lastName: `%${searchCriteria}%`, + email: `%${searchCriteria}%`, + }, + ) + .take(pageSize) + .skip((currentPage - 1 ) * pageSize) + .getManyAndCount() + } + + async findBySearchCriteriaPagedNotActivated(searchCriteria: string, currentPage: number, pageSize: number): Promise<[User[], number]> { + return await this.createQueryBuilder('user') + .where( + 'user.firstName like :name or user.lastName like :lastName or user.email like :email', + { + name: `%${searchCriteria}%`, + lastName: `%${searchCriteria}%`, + email: `%${searchCriteria}%`, + emailChecked: false + }, + ) + .take(pageSize) + .skip((currentPage - 1 ) * pageSize) + .getManyAndCount() + } } From cdd31ccba9e0ea41cc1aee82c6abbd6ca17a8dd2 Mon Sep 17 00:00:00 2001 From: Einhornimmond Date: Sat, 12 Feb 2022 12:41:13 +0100 Subject: [PATCH 2/6] fix nested where conditions --- backend/src/typeorm/repository/User.ts | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/backend/src/typeorm/repository/User.ts b/backend/src/typeorm/repository/User.ts index bf9d55587..e4efedaf7 100644 --- a/backend/src/typeorm/repository/User.ts +++ b/backend/src/typeorm/repository/User.ts @@ -1,4 +1,4 @@ -import { EntityRepository, Repository } from '@dbTools/typeorm' +import { Brackets, EntityRepository, Repository } from '@dbTools/typeorm' import { User } from '@entity/User' import internal from 'stream' @@ -60,15 +60,22 @@ export class UserRepository extends Repository { } async findBySearchCriteriaPagedNotActivated(searchCriteria: string, currentPage: number, pageSize: number): Promise<[User[], number]> { + return await this.createQueryBuilder('user') .where( - 'user.firstName like :name or user.lastName like :lastName or user.email like :email', - { - name: `%${searchCriteria}%`, - lastName: `%${searchCriteria}%`, - email: `%${searchCriteria}%`, - emailChecked: false - }, + new Brackets(qb => { + qb.where( + 'user.firstName like :name or user.lastName like :lastName or user.email like :email', + { + name: `%${searchCriteria}%`, + lastName: `%${searchCriteria}%`, + email: `%${searchCriteria}%`, + }, + ) + }) + ) + .andWhere( + {emailChecked: false} ) .take(pageSize) .skip((currentPage - 1 ) * pageSize) From 692433ce51a47b90506a82d05bda1c48f971559c Mon Sep 17 00:00:00 2001 From: Einhornimmond Date: Sun, 13 Feb 2022 09:22:05 +0100 Subject: [PATCH 3/6] try fix linting --- backend/src/graphql/resolver/AdminResolver.ts | 16 +++++++++--- backend/src/typeorm/repository/User.ts | 26 ++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/backend/src/graphql/resolver/AdminResolver.ts b/backend/src/graphql/resolver/AdminResolver.ts index 2b6138dee..fbd3b04fa 100644 --- a/backend/src/graphql/resolver/AdminResolver.ts +++ b/backend/src/graphql/resolver/AdminResolver.ts @@ -34,12 +34,20 @@ export class AdminResolver { const userRepository = getCustomRepository(UserRepository) let users: dbUser[] let count: number - if(notActivated) { - [users, count] = await userRepository.findBySearchCriteriaPagedNotActivated(searchText, currentPage, pageSize) + if (notActivated) { + [users, count] = await userRepository.findBySearchCriteriaPagedNotActivated( + searchText, + currentPage, + pageSize, + ) } else { - [users, count] = await userRepository.findBySearchCriteriaPaged(searchText, currentPage, pageSize) + [users, count] = await userRepository.findBySearchCriteriaPaged( + searchText, + currentPage, + pageSize, + ) } - let adminUsers = await Promise.all( + const adminUsers = await Promise.all( users.map(async (user) => { const adminUser = new UserAdmin() adminUser.userId = user.id diff --git a/backend/src/typeorm/repository/User.ts b/backend/src/typeorm/repository/User.ts index e4efedaf7..e43c29e22 100644 --- a/backend/src/typeorm/repository/User.ts +++ b/backend/src/typeorm/repository/User.ts @@ -44,7 +44,12 @@ export class UserRepository extends Repository { ) .getMany() } - async findBySearchCriteriaPaged(searchCriteria: string, currentPage: number, pageSize: number): Promise<[User[], number]> { + + async findBySearchCriteriaPaged( + searchCriteria: string, + currentPage: number, + pageSize: number, + ): Promise<[User[], number]> { return await this.createQueryBuilder('user') .where( 'user.firstName like :name or user.lastName like :lastName or user.email like :email', @@ -55,15 +60,18 @@ export class UserRepository extends Repository { }, ) .take(pageSize) - .skip((currentPage - 1 ) * pageSize) + .skip((currentPage - 1) * pageSize) .getManyAndCount() } - async findBySearchCriteriaPagedNotActivated(searchCriteria: string, currentPage: number, pageSize: number): Promise<[User[], number]> { - + async findBySearchCriteriaPagedNotActivated( + searchCriteria: string, + currentPage: number, + pageSize: number, + ): Promise<[User[], number]> { return await this.createQueryBuilder('user') .where( - new Brackets(qb => { + new Brackets((qb) => { qb.where( 'user.firstName like :name or user.lastName like :lastName or user.email like :email', { @@ -72,13 +80,11 @@ export class UserRepository extends Repository { email: `%${searchCriteria}%`, }, ) - }) - ) - .andWhere( - {emailChecked: false} + }), ) + .andWhere({ emailChecked: false }) .take(pageSize) - .skip((currentPage - 1 ) * pageSize) + .skip((currentPage - 1) * pageSize) .getManyAndCount() } } From 84d7f17d8502af189882e312a050bc0c30df1ea3 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 13 Feb 2022 13:33:44 +0100 Subject: [PATCH 4/6] Update backend/src/typeorm/repository/User.ts Co-authored-by: Ulf Gebhardt --- backend/src/typeorm/repository/User.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/typeorm/repository/User.ts b/backend/src/typeorm/repository/User.ts index e43c29e22..6045c402a 100644 --- a/backend/src/typeorm/repository/User.ts +++ b/backend/src/typeorm/repository/User.ts @@ -1,6 +1,5 @@ import { Brackets, EntityRepository, Repository } from '@dbTools/typeorm' import { User } from '@entity/User' -import internal from 'stream' @EntityRepository(User) export class UserRepository extends Repository { From b3523b4c631c4f507ec612715062e40f0bc14a3f Mon Sep 17 00:00:00 2001 From: Einhornimmond Date: Mon, 21 Feb 2022 12:03:19 +0100 Subject: [PATCH 5/6] generic filtering, prevent overfetching from db --- backend/src/graphql/resolver/AdminResolver.ts | 40 +++++++++++------- backend/src/typeorm/repository/User.ts | 41 +++---------------- 2 files changed, 32 insertions(+), 49 deletions(-) diff --git a/backend/src/graphql/resolver/AdminResolver.ts b/backend/src/graphql/resolver/AdminResolver.ts index 937ed8567..4a2e59644 100644 --- a/backend/src/graphql/resolver/AdminResolver.ts +++ b/backend/src/graphql/resolver/AdminResolver.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { Resolver, Query, Arg, Args, Authorized, Mutation, Ctx } from 'type-graphql' -import { getCustomRepository, Raw } from '@dbTools/typeorm' +import { getCustomRepository, ObjectLiteral, Raw } from '@dbTools/typeorm' import { UserAdmin, SearchUsersResult } from '../model/UserAdmin' import { PendingCreation } from '../model/PendingCreation' import { CreatePendingCreations } from '../model/CreatePendingCreations' @@ -27,6 +27,13 @@ import { LoginEmailOptIn } from '@entity/LoginEmailOptIn' // const EMAIL_OPT_IN_REGISTER = 1 // const EMAIL_OPT_UNKNOWN = 3 // elopage? +/* +parent:any, +args:any, +context:any, +info:any +*/ + @Resolver() export class AdminResolver { @Authorized([RIGHTS.SEARCH_USERS]) @@ -35,21 +42,25 @@ export class AdminResolver { @Args() { searchText, currentPage = 1, pageSize = 25, notActivated = false }: SearchUsersArgs, ): Promise { const userRepository = getCustomRepository(UserRepository) - let users: dbUser[] - let count: number + + const filterCriteria: ObjectLiteral[] = [] if (notActivated) { - [users, count] = await userRepository.findBySearchCriteriaPagedNotActivated( - searchText, - currentPage, - pageSize, - ) - } else { - [users, count] = await userRepository.findBySearchCriteriaPaged( - searchText, - currentPage, - pageSize, - ) + filterCriteria.push({ emailChecked: false }) } + // prevent overfetching data from db, select only needed columns + // prevent reading and transmitting data from db at least 300 Bytes + // one of my example dataset shrink down from 342 Bytes to 42 Bytes, that's ~88% saved db bandwith + const userFields = ['id', 'firstName', 'lastName', 'email', 'emailChecked'] + const [users, count] = await userRepository.findBySearchCriteriaPagedFiltered( + userFields.map((fieldName) => { + return 'user.' + fieldName + }), + searchText, + filterCriteria, + currentPage, + pageSize, + ) + const adminUsers = await Promise.all( users.map(async (user) => { const adminUser = new UserAdmin() @@ -70,6 +81,7 @@ export class AdminResolver { updatedAt: 'DESC', createdAt: 'DESC', }, + select: ['updatedAt', 'createdAt'], }, ) if (emailOptIn) { diff --git a/backend/src/typeorm/repository/User.ts b/backend/src/typeorm/repository/User.ts index 6045c402a..e7ef7f0bc 100644 --- a/backend/src/typeorm/repository/User.ts +++ b/backend/src/typeorm/repository/User.ts @@ -1,4 +1,4 @@ -import { Brackets, EntityRepository, Repository } from '@dbTools/typeorm' +import { Brackets, EntityRepository, ObjectLiteral, Repository } from '@dbTools/typeorm' import { User } from '@entity/User' @EntityRepository(User) @@ -31,44 +31,15 @@ export class UserRepository extends Repository { return usersIndiced } - async findBySearchCriteria(searchCriteria: string): Promise { - return await this.createQueryBuilder('user') - .where( - 'user.firstName like :name or user.lastName like :lastName or user.email like :email', - { - name: `%${searchCriteria}%`, - lastName: `%${searchCriteria}%`, - email: `%${searchCriteria}%`, - }, - ) - .getMany() - } - - async findBySearchCriteriaPaged( - searchCriteria: string, - currentPage: number, - pageSize: number, - ): Promise<[User[], number]> { - return await this.createQueryBuilder('user') - .where( - 'user.firstName like :name or user.lastName like :lastName or user.email like :email', - { - name: `%${searchCriteria}%`, - lastName: `%${searchCriteria}%`, - email: `%${searchCriteria}%`, - }, - ) - .take(pageSize) - .skip((currentPage - 1) * pageSize) - .getManyAndCount() - } - - async findBySearchCriteriaPagedNotActivated( + async findBySearchCriteriaPagedFiltered( + select: string[], searchCriteria: string, + filterCriteria: ObjectLiteral[], currentPage: number, pageSize: number, ): Promise<[User[], number]> { return await this.createQueryBuilder('user') + .select(select) .where( new Brackets((qb) => { qb.where( @@ -81,7 +52,7 @@ export class UserRepository extends Repository { ) }), ) - .andWhere({ emailChecked: false }) + .andWhere(filterCriteria) .take(pageSize) .skip((currentPage - 1) * pageSize) .getManyAndCount() From c3d3929ac644561167c9b8460a0aa823b6378b4b Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 21 Feb 2022 13:23:04 +0100 Subject: [PATCH 6/6] Update backend/src/graphql/resolver/AdminResolver.ts Co-authored-by: Ulf Gebhardt --- backend/src/graphql/resolver/AdminResolver.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/backend/src/graphql/resolver/AdminResolver.ts b/backend/src/graphql/resolver/AdminResolver.ts index 8a333a233..f2fb00666 100644 --- a/backend/src/graphql/resolver/AdminResolver.ts +++ b/backend/src/graphql/resolver/AdminResolver.ts @@ -28,13 +28,6 @@ import { User } from '@entity/User' // const EMAIL_OPT_IN_REGISTER = 1 // const EMAIL_OPT_UNKNOWN = 3 // elopage? -/* -parent:any, -args:any, -context:any, -info:any -*/ - @Resolver() export class AdminResolver { @Authorized([RIGHTS.SEARCH_USERS])