diff --git a/docker-compose.override.yml b/docker-compose.override.yml index e4bc82a14..39797a219 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -49,10 +49,6 @@ services: context: . target: community_server dockerfile: ./community_server/Dockerfile - environment: - - DB_PASSWORD='' - - DB_USER='root' - - DB_DATABASE='gradido_community' depends_on: - mariadb networks: diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 000000000..d8f307dfe --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,96 @@ +version: "3.4" + +services: + + ######################################################### + ## MARIADB ############################################## + ######################################################### + mariadb: + build: + context: . + dockerfile: ./mariadb/Dockerfile + target: mariadb_server_test + environment: + - MARIADB_ALLOW_EMPTY_PASSWORD=1 + - MARIADB_USER=root + networks: + - internal-net + ports: + - 3306:3306 + volumes: + - db_test_vol:/var/lib/mysql + + ######################################################### + ## LOGIN SERVER ######################################### + ######################################################### + login-server: + build: + context: ./login_server/ + target: test + security_opt: + - seccomp:unconfined + cap_add: + - SYS_PTRACE + depends_on: + - mariadb + networks: + - internal-net + volumes: + - ./logs:/var/log/grd_login + - ./configs/login_server:/etc/grd_login + - ~/coverage/login:/code/build/coverage + + ######################################################### + ## COMMUNITY SERVER (cakephp with php-fpm) ############## + ######################################################### + community-server: + build: + context: . + target: test + dockerfile: ./community_server/Dockerfile + depends_on: + - mariadb + networks: + - internal-net + volumes: + - ./community_server/config/php-fpm/php-ini-overrides.ini:/etc/php/7.4/fpm/conf.d/99-overrides.ini + - ./community_server/src:/var/www/cakephp/src + - ~/coverage/community:/var/www/cakephp/webroot/coverage + + + ######################################################### + ## PHPMYADMIN ########################################### + ######################################################### + phpmyadmin: + image: phpmyadmin + environment: + - PMA_ARBITRARY=1 + #restart: always + ports: + - 8074:80 + networks: + - internal-net + - external-net + volumes: + - /sessions + + ######################################################### + ## skeema for updating dbs if changes happend ########### + ######################################################### + skeema: + build: + context: . + dockerfile: ./skeema/Dockerfile + target: skeema_run + depends_on: + - mariadb + networks: + - internal-net + +networks: + external-net: + internal-net: + internal: true + +volumes: + db_test_vol: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7e9ee3bc1..e1a1b8dc5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ services: build: context: . dockerfile: ./mariadb/Dockerfile - target: mariadb_server + target: mariadb_server_test environment: - MARIADB_ALLOW_EMPTY_PASSWORD=1 - MARIADB_USER=root @@ -47,7 +47,7 @@ services: ports: - 3306:3306 volumes: - - db_vol:/var/lib/mysql + - db_vol:/var/lib/mysql ######################################################### ## LOGIN SERVER ######################################### diff --git a/login_server/Dockerfile b/login_server/Dockerfile index def7875ec..a933ce28f 100644 --- a/login_server/Dockerfile +++ b/login_server/Dockerfile @@ -2,11 +2,13 @@ ######################################################################################################### # Prepare debug ######################################################################################################### -FROM gradido/login_dependencies:alpine-debug-2 as prepare_debug +FROM gradido/login_dependencies:gcc9-debug-1 as prepare_debug ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} +RUN echo '/usr/local/lib' >> /etc/ld.so.conf && ldconfig + COPY ./CMakeLists.txt.lib ./CMakeLists.txt RUN ln -s /usr/local/googletest ./googletest COPY ./src ./src @@ -24,21 +26,23 @@ FROM prepare_debug as test ENV DOCKER_WORKDIR="/code" WORKDIR ${DOCKER_WORKDIR} -RUN apk add --no-cache bash perl -#RUN git clone https://github.com/linux-test-project/lcov.git --branch=v1.15 && \ - # cd lcov && \ - # make install +#RUN apt-get update && \ +# apt-get install -y --no-install-recommends gdb && \ +# apt-get autoclean && \ +# apt-get autoremove && \ +# apt-get clean && \ +# rm -rf /var/lib/apt/lists/* RUN mkdir build && cd build && \ cmake -DCMAKE_BUILD_TYPE=Debug .. && \ make -j$(nproc) Gradido_LoginServer_Test RUN chmod +x build/bin/Gradido_LoginServer_Test -#CMD [ "./build/bin/Gradido_LoginServer_Test" ] -#ENTRYPOINT [ "/usr/bin/make", "-C", "/code/build", "coverage" ] +#CMD gdb -ex run ./build/bin/Gradido_LoginServer_Test +CMD ./build/bin/Gradido_LoginServer_Test -ENTRYPOINT make -C build coverage +#ENTRYPOINT make -C build coverage ######################################################################################################### # Build debug diff --git a/login_server/Dockerfile.alpine-debug b/login_server/Dockerfile.alpine-debug index b9a5b14d0..59ab8d39e 100644 --- a/login_server/Dockerfile.alpine-debug +++ b/login_server/Dockerfile.alpine-debug @@ -1,4 +1,4 @@ -FROM gradido/login_dependencies:alpine-debug-1 as login_server_alpine_debug +FROM gradido/login_dependencies:alpine-debug-2 as login_server_alpine_debug ENV DOCKER_WORKDIR="/code" @@ -10,6 +10,7 @@ WORKDIR ${DOCKER_WORKDIR} COPY ./CMakeLists.txt.lib ./CMakeLists.txt COPY ./scripts ./scripts COPY ./cmake ./cmake +RUN ln -s /usr/local/googletest ./googletest CMD cd scripts; ./build_debug.sh; cd ..; ./build/bin/Gradido_LoginServer diff --git a/login_server/Dockerfiles/Dockerfile.dependencies b/login_server/Dockerfiles/Dockerfile.dependencies index ba275ca23..24fd31691 100644 --- a/login_server/Dockerfiles/Dockerfile.dependencies +++ b/login_server/Dockerfiles/Dockerfile.dependencies @@ -1,11 +1,8 @@ -######################################################################################################### -# gcc 9 with libssl -######################################################################################################### -FROM gcc:9 as gcc_9_libssl -USER root +##### BUILD-ENV ##### +FROM gcc:9 as gcc9_build RUN apt-get update && \ - apt-get install -y --no-install-recommends libssl-dev libboost-dev && \ + apt-get install -y --no-install-recommends libssl-dev libboost-dev gettext libsodium-dev lcov && \ apt-get autoclean && \ apt-get autoremove && \ apt-get clean && \ @@ -15,80 +12,74 @@ RUN apt-get update && \ ######################################################################################################### # gcc 9 cmake ######################################################################################################### -FROM gcc_9_libssl as gcc_9_cmake - -USER root -ENV DOCKER_WORKDIR="/code" - -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -COPY ./cmake ./cmake +FROM gcc9_build as gcc_9_cmake RUN git clone https://github.com/Kitware/CMake.git --branch=v3.19.8 && \ cd CMake && \ ./bootstrap --parallel=$(nproc) && \ make -j$(nproc) && \ - make install && \ - cd .. && \ - rm -rf CMake + make install + +######### BUILD grpc ############## +FROM gcc_9_cmake as gcc_9_grpc + +ARG BUILD_TYPE=Debug + +RUN git clone https://github.com/grpc/grpc.git --branch=v1.37.0 --recursive -j4 && \ + cd grpc && \ + mkdir build && cd build && \ + cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. && make -j$(nproc) && \ + make install + +# abseil don't install themself correctly +RUN cp -r grpc/third_party/abseil-cpp/absl /usr/local/include/ + +# protobuf libs missing after make install +RUN cp grpc/build/third_party/protobuf/*.a /usr/local/lib/ + +######### BUILD poco ############## +FROM gcc_9_cmake as gcc_9_poco + +ARG BUILD_TYPE=Debug + +RUN git clone https://github.com/pocoproject/poco.git --recursive && \ + cd poco && \ + git checkout poco-1.9.4-release && \ + mkdir cmake-build && cd cmake-build && \ + cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. && make -j$(nproc) && \ + make install + +######### BUILD mariadb ########### +FROM gcc_9_cmake as gcc_9_mariadb-connector + +ARG BUILD_TYPE=Debug + +RUN git clone https://github.com/mariadb-corporation/mariadb-connector-c.git && \ + cd mariadb-connector-c && \ + git checkout 159540f && \ + mkdir build && cd build && \ + cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. && make -j$(nproc) && \ + make install ######################################################################################################### -# debug build preparation +# builded libs and binaries ######################################################################################################### -FROM gcc_9_libssl as build_debug_dependencies +FROM gcc9_build as gcc9_libs -USER root -ENV DOCKER_WORKDIR="/code" +RUN echo '/usr/local/lib' >> /etc/ld.so.conf && ldconfig -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -# copy CMake from last stage +# copy CMake from cmake stage COPY --from=gcc_9_cmake /usr/local/bin/cmake /usr/local/bin/cmake COPY --from=gcc_9_cmake /usr/local/share/cmake-3.19/Modules /usr/local/share/cmake-3.19/Modules COPY --from=gcc_9_cmake /usr/local/share/cmake-3.19/Templates /usr/local/share/cmake-3.19/Templates -COPY ./dependencies ./dependencies -COPY ./scripts ./scripts -COPY ./CMakeLists.txt . -COPY ./src ./src +# copy from grpc +COPY --from=gcc_9_grpc /usr/local /usr/local +COPY --from=gcc_9_grpc /grpc/third_party/googletest /usr/local/googletest +# COPY from poco +COPY --from=gcc_9_poco /usr/local /usr/local -RUN cd scripts && \ - chmod +x ./prepare_build.sh && \ - ./prepare_build.sh && \ - mkdir ../build && \ - chmod +x ./build_debug.sh && \ - ./build_debug.sh - -# remove unneccessary stuff -RUN rm -rf build/bin/Gradido_LoginServer - -######################################################################################################### -# debug build -######################################################################################################### -FROM gcc_9_libssl as build_debug - -USER root -ENV DOCKER_WORKDIR="/code" - -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - - -# copy CMake from last stage -COPY --from=build_debug_dependencies /usr/local/bin/cmake /usr/local/bin/cmake -COPY --from=build_debug_dependencies /usr/local/share/cmake-3.19 /usr/local/share/cmake-3.19 - -COPY --from=build_debug_dependencies /code/build/bin /code/build/bin -COPY --from=build_debug_dependencies /code/build/lib /code/build/lib - -# grpc -COPY --from=build_debug_dependencies /code/build/dependencies/grpc/lib /build/dependencies/grpc/lib -COPY --from=build_debug_dependencies /code/build/dependencies/grpc/third_party/protobuf/lib /build/dependencies/grpc/third_party/protobuf/lib -COPY --from=build_debug_dependencies /code/build/dependencies/grpc/third_party/re2/lib /build/dependencies/grpc/third_party/re2/lib -COPY --from=build_debug_dependencies /code/build/dependencies/grpc/third_party/zlib/lib /build/dependencies/grpc/third_party/zlib/lib -COPY --from=build_debug_dependencies /code/build/dependencies/grpc/third_party/cares/cares/lib /build/dependencies/grpc/third_party/cares/cares/lib -COPY --from=build_debug_dependencies /code/build/dependencies/mariadb-connector-c/libmariadb/lib /build/dependencies/mariadb-connector-c/libmariadb/lib +# COPY from mariadb +COPY --from=gcc_9_mariadb-connector /usr/local /usr/local diff --git a/login_server/src/cpp/SingletonManager/ConnectionManager.cpp b/login_server/src/cpp/SingletonManager/ConnectionManager.cpp index 5e5bc4935..48db3d29d 100644 --- a/login_server/src/cpp/SingletonManager/ConnectionManager.cpp +++ b/login_server/src/cpp/SingletonManager/ConnectionManager.cpp @@ -57,6 +57,7 @@ bool ConnectionManager::setConnectionsFromConfig(const Poco::Util::LayeredConfig dbConfig << "user=" << config.getString(firstKeyPart + ".db.user", "root") << ";"; dbConfig << "password=" << config.getString(firstKeyPart + ".db.password", "") << ";"; dbConfig << "auto-reconnect=true"; + std::clog << "try connect with: " << dbConfig.str() << std::endl; setConnection(dbConfig.str(), type); diff --git a/login_server/src/cpp/tasks/CPUSheduler.cpp b/login_server/src/cpp/tasks/CPUSheduler.cpp index b1c71c0ce..affa5ce82 100644 --- a/login_server/src/cpp/tasks/CPUSheduler.cpp +++ b/login_server/src/cpp/tasks/CPUSheduler.cpp @@ -11,7 +11,14 @@ namespace UniLib { CPUSheduler::CPUSheduler(uint8_t threadCount, const char* name) : mThreads(new CPUShedulerThread*[threadCount]), mThreadCount(threadCount), mName(name) { - char nameBuffer[10]; memset(nameBuffer, 0, 10); + char static_nameBuffer[10]; memset(static_nameBuffer, 0, 10); + char* nameBuffer = static_nameBuffer; + if(threadCount > 99) { + int bufferSize = 7 + strlen(std::to_string(threadCount).data())+1; + nameBuffer = (char*)malloc(bufferSize); + memset(nameBuffer, 0, bufferSize); + + } //uint8_t len = std:: min(strlen(name), 7); uint8_t len = strlen(name); if(len > 7) len = 7; @@ -24,6 +31,9 @@ namespace UniLib { sprintf(&nameBuffer[len], "%.2d", i); mThreads[i] = new CPUShedulerThread(this, nameBuffer); } + if(threadCount > 99) { + free(nameBuffer); + } } CPUSheduler::~CPUSheduler() diff --git a/login_server/src/cpp/test/TestTasks.cpp b/login_server/src/cpp/test/TestTasks.cpp index c7d0f33ab..805227b7b 100644 --- a/login_server/src/cpp/test/TestTasks.cpp +++ b/login_server/src/cpp/test/TestTasks.cpp @@ -3,6 +3,7 @@ #include "Poco/Environment.h" #include #include +#include #include "Poco/Logger.h" #include "Poco/Path.h" @@ -87,27 +88,41 @@ int TestTasks::init() int TestTasks::test() { + std::clog << "start with task test" << std::endl; auto workerCount = Poco::Environment::processorCount() * 4; auto taskCount = workerCount + workerCount * (randombytes_random() % 4); - printf("[TestTasks::test] taskCount: %d\n", taskCount); + std::clog << "worker count: " << std::to_string(workerCount) << ", task count: " << std::to_string(taskCount) << std::endl; for (int i = 1; i <= taskCount; i++) { Poco::AutoPtr task = new RandomCPUTask(&mTaskScheduler, this, i); lock(); mTasks.insert(std::pair(i, task)); unlock(); task->scheduleTask(task); + //std::clog << "start task: " << std::to_string(i) << std::endl; } + + std::clog << "all tasks started" << std::endl; int maxWaitCylces = 3000; bool finished = false; do { maxWaitCylces--; Poco::Thread::sleep(5); - lock(); - if (mTasks.size() == 0) { - finished = true; + if(mErrors.size() > 0) { + std::clog << std::to_string(mErrors.size()) << " errors" << std::endl; + } + try { + lock(); + if (mTasks.size() == 0) { + finished = true; + } + unlock(); + } catch(Poco::Exception& ex) { + std::clog << "Poco Exception while waiting on tasks: " << ex.displayText() << std::endl; + } catch(std::exception& ex) { + std::clog << "std::exception while waiting on tasks: " << ex.what() << std::endl; } - unlock(); } while (!finished && maxWaitCylces > 0); + std::clog << "all tasks now finished" << std::endl; lock(); bool hasErrors = false; @@ -150,4 +165,5 @@ void TestTasks::releaseTask(int nr) mErrors.push_back("[TestTasks] task entry not found" + std::to_string(nr)); } unlock(); + } \ No newline at end of file diff --git a/login_server/src/cpp/test/main.cpp b/login_server/src/cpp/test/main.cpp index f1bbe196b..933d555d3 100644 --- a/login_server/src/cpp/test/main.cpp +++ b/login_server/src/cpp/test/main.cpp @@ -6,6 +6,11 @@ #include "Poco/Util/PropertyFileConfiguration.h" #include "Poco/Environment.h" #include "Poco/Path.h" +#include "Poco/AsyncChannel.h" +#include "Poco/SimpleFileChannel.h" +#include "Poco/FileChannel.h" +#include "Poco/ConsoleChannel.h" +#include "Poco/SplitterChannel.h" #include "../SingletonManager/ConnectionManager.h" @@ -118,6 +123,28 @@ int load(int argc, char* argv[]) { } } + std::string log_Path = "/var/log/grd_login/"; +//#ifdef _WIN32 +#if defined(_WIN32) || defined(_WIN64) + log_Path = "./"; +#endif + + std::string filePath = log_Path + "errorLog.txt"; + Poco::AutoPtr logConsoleChannel(new Poco::ConsoleChannel); + Poco::AutoPtr logFileChannel(new Poco::FileChannel(filePath)); + Poco::AutoPtr logSplitter(new Poco::SplitterChannel); + logSplitter->addChannel(logConsoleChannel); + logSplitter->addChannel(logFileChannel); + + Poco::AutoPtr logAsyncChannel(new Poco::AsyncChannel(logSplitter)); + + Poco::Logger& log = Poco::Logger::get("errorLog"); + log.setChannel(logAsyncChannel); + log.setLevel("information"); + + log.error("Test Error"); + + //errorLog //printf("try connect php server mysql \n"); //conn->setConnectionsFromConfig(*test_config, CONNECTION_MYSQL_PHP_SERVER); @@ -163,10 +190,12 @@ int load(int argc, char* argv[]) { << "(1, 3, 'gdd_test_topic', 1, 0, 1, NULL, NULL, '1999-12-31 23:00:00', 0, '2020-09-14 18:29:04'); "; runMysql(ss.str()); ss.str(std::string()); + std::clog << "after inserting everything in db" << std::endl; fillTests(); for (std::list::iterator it = gTests.begin(); it != gTests.end(); it++) { + std::clog << "call init on test: " << (*it)->getName() << std::endl; if ((*it)->init()) printf("Fehler bei Init test: %s\n", (*it)->getName()); } return 0; @@ -179,7 +208,11 @@ int run() { //printf("running: %s\n", it->getName()); printf("running test: %s\n", (*it)->getName()); - if (!(*it)->test()) printf("success\n"); + try { + if (!(*it)->test()) printf("success\n"); + } catch(std::exception& ex) { + std::clog << "exception in running test: " << ex.what() << std::endl; + } } return 0; } @@ -207,6 +240,9 @@ int main(int argc, char** argv) } catch(std::exception& ex) { printf("no catched exception while loading: %s\n", ex.what()); } + + //printf ("\nStack Limit = %ld and %ld max\n", limit.rlim_cur, limit.rlim_max); + run(); ende(); ::testing::InitGoogleTest(&argc, argv); diff --git a/skeema/Dockerfile b/skeema/Dockerfile index 0c754899d..f9b0a339c 100644 --- a/skeema/Dockerfile +++ b/skeema/Dockerfile @@ -9,7 +9,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /go/bin/skeema . ######################################################################################################### # Run skeema ######################################################################################################### -From alpine:latest as skeema_run +FROM alpine:3.13.5 as skeema_run ENV DOCKER_WORKDIR="/skeema"