diff --git a/backend/package.json b/backend/package.json index f6b66b07e..786fe6641 100644 --- a/backend/package.json +++ b/backend/package.json @@ -69,6 +69,7 @@ "helmet": "~3.22.0", "ioredis": "^4.16.1", "jsonwebtoken": "~8.5.1", + "languagedetect": "^2.0.0", "linkifyjs": "~2.1.8", "lodash": "~4.17.14", "merge-graphql-schemas": "^1.7.7", diff --git a/backend/src/middleware/index.js b/backend/src/middleware/index.js index 83b0104ec..0ad8cb1ae 100644 --- a/backend/src/middleware/index.js +++ b/backend/src/middleware/index.js @@ -14,6 +14,7 @@ import notifications from './notifications/notificationsMiddleware' import hashtags from './hashtags/hashtagsMiddleware' import email from './email/emailMiddleware' import sentry from './sentryMiddleware' +import languages from './languages/languages' export default (schema) => { const middlewares = { @@ -30,6 +31,7 @@ export default (schema) => { softDelete, includedFields, orderBy, + languages, } let order = [ @@ -39,6 +41,7 @@ export default (schema) => { // 'activityPub', disabled temporarily 'validation', 'sluggify', + 'languages', 'excerpt', 'email', 'notifications', diff --git a/backend/src/middleware/languages/languages.js b/backend/src/middleware/languages/languages.js new file mode 100644 index 000000000..3cf760f31 --- /dev/null +++ b/backend/src/middleware/languages/languages.js @@ -0,0 +1,28 @@ +import LanguageDetect from 'languagedetect' +import sanitizeHtml from 'sanitize-html' + +const removeHtmlTags = (input) => { + return sanitizeHtml(input, { + allowedTags: [], + allowedAttributes: {}, + }) +} + +const setPostLanguage = (text) => { + const lngDetector = new LanguageDetect() + lngDetector.setLanguageType('iso2') + return lngDetector.detect(removeHtmlTags(text), 1)[0][0] +} + +export default { + Mutation: { + CreatePost: async (resolve, root, args, context, info) => { + args.language = await setPostLanguage(args.content) + return resolve(root, args, context, info) + }, + UpdatePost: async (resolve, root, args, context, info) => { + args.language = await setPostLanguage(args.content) + return resolve(root, args, context, info) + }, + }, +} diff --git a/backend/src/middleware/languages/languages.spec.js b/backend/src/middleware/languages/languages.spec.js new file mode 100644 index 000000000..1448e4e4b --- /dev/null +++ b/backend/src/middleware/languages/languages.spec.js @@ -0,0 +1,132 @@ +import Factory, { cleanDatabase } from '../../db/factories' +import { gql } from '../../helpers/jest' +import { getNeode, getDriver } from '../../db/neo4j' +import createServer from '../../server' +import { createTestClient } from 'apollo-server-testing' + +let mutate +let authenticatedUser +let variables + +const driver = getDriver() +const neode = getNeode() + +beforeAll(async () => { + const { server } = createServer({ + context: () => { + return { + driver, + neode, + user: authenticatedUser, + } + }, + }) + mutate = createTestClient(server).mutate +}) + +afterAll(async () => { + await cleanDatabase() +}) + +const createPostMutation = gql` + mutation($title: String!, $content: String!, $categoryIds: [ID]) { + CreatePost(title: $title, content: $content, categoryIds: $categoryIds) { + language + } + } +` + +describe('languagesMiddleware', () => { + variables = { + title: 'Test post languages', + categoryIds: ['cat9'], + } + + beforeAll(async () => { + await cleanDatabase() + const user = await Factory.build('user') + authenticatedUser = await user.toJson() + await Factory.build('category', { + id: 'cat9', + name: 'Democracy & Politics', + icon: 'university', + }) + }) + + it('detects German', async () => { + variables = { + ...variables, + content: 'Jeder sollte vor seiner eigenen Tür kehren.', + } + await expect( + mutate({ + mutation: createPostMutation, + variables, + }), + ).resolves.toMatchObject({ + data: { + CreatePost: { + language: 'de', + }, + }, + }) + }) + + it('detects English', async () => { + variables = { + ...variables, + content: 'A journey of a thousand miles begins with a single step.', + } + await expect( + mutate({ + mutation: createPostMutation, + variables, + }), + ).resolves.toMatchObject({ + data: { + CreatePost: { + language: 'en', + }, + }, + }) + }) + + it('detects Spanish', async () => { + variables = { + ...variables, + content: 'A caballo regalado, no le mires el diente.', + } + await expect( + mutate({ + mutation: createPostMutation, + variables, + }), + ).resolves.toMatchObject({ + data: { + CreatePost: { + language: 'es', + }, + }, + }) + }) + + it('detects German in between lots of html tags', async () => { + variables = { + ...variables, + content: + 'Jeder sollte vor seiner eigenen
Tür
kehren.', + } + await expect( + mutate({ + mutation: createPostMutation, + variables, + }), + ).resolves.toMatchObject({ + data: { + CreatePost: { + language: 'de', + }, + }, + }) + }) +}) diff --git a/backend/src/middleware/slugify/uniqueSlug.js b/backend/src/middleware/slugify/uniqueSlug.js index ca37cd562..7cfb89c19 100644 --- a/backend/src/middleware/slugify/uniqueSlug.js +++ b/backend/src/middleware/slugify/uniqueSlug.js @@ -2,6 +2,7 @@ import slugify from 'slug' export default async function uniqueSlug(string, isUnique) { const slug = slugify(string || 'anonymous', { lower: true, + multicharmap: { Ä: 'AE', ä: 'ae', Ö: 'OE', ö: 'oe', Ü: 'UE', ü: 'ue', ß: 'ss' }, }) if (await isUnique(slug)) return slug diff --git a/backend/src/middleware/slugify/uniqueSlug.spec.js b/backend/src/middleware/slugify/uniqueSlug.spec.js index ff14a56ef..d002eae03 100644 --- a/backend/src/middleware/slugify/uniqueSlug.spec.js +++ b/backend/src/middleware/slugify/uniqueSlug.spec.js @@ -18,4 +18,16 @@ describe('uniqueSlug', () => { const isUnique = jest.fn().mockResolvedValue(true) expect(uniqueSlug(string, isUnique)).resolves.toEqual('anonymous') }) + + it('Converts umlaut to a two letter equivalent', async () => { + const umlaut = 'ÄÖÜäöüß' + const isUnique = jest.fn().mockResolvedValue(true) + await expect(uniqueSlug(umlaut, isUnique)).resolves.toEqual('aeoeueaeoeuess') + }) + + it('Removes Spanish enya and diacritics', async () => { + const diacritics = 'áàéèíìóòúùñçÁÀÉÈÍÌÓÒÚÙÑÇ' + const isUnique = jest.fn().mockResolvedValue(true) + await expect(uniqueSlug(diacritics, isUnique)).resolves.toEqual('aaeeiioouuncaaeeiioouunc') + }) }) diff --git a/backend/src/schema/resolvers/posts.spec.js b/backend/src/schema/resolvers/posts.spec.js index b24383fba..f0c57b8fb 100644 --- a/backend/src/schema/resolvers/posts.spec.js +++ b/backend/src/schema/resolvers/posts.spec.js @@ -317,19 +317,6 @@ describe('CreatePost', () => { expected, ) }) - - describe('language', () => { - beforeEach(() => { - variables = { ...variables, language: 'es' } - }) - - it('allows a user to set the language of the post', async () => { - const expected = { data: { CreatePost: { language: 'es' } } } - await expect(mutate({ mutation: createPostMutation, variables })).resolves.toMatchObject( - expected, - ) - }) - }) }) }) diff --git a/backend/yarn.lock b/backend/yarn.lock index 0bbc62515..7d6558da0 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -6302,6 +6302,11 @@ knuth-shuffle-seeded@^1.0.6: dependencies: seed-random "~2.2.0" +languagedetect@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/languagedetect/-/languagedetect-2.0.0.tgz#4b8fa2b7593b2a3a02fb1100891041c53238936c" + integrity sha512-AZb/liiQ+6ZoTj4f1J0aE6OkzhCo8fyH+tuSaPfSo8YHCWLFJrdSixhtO2TYdIkjcDQNaR4RmGaV2A5FJklDMQ== + latest-version@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-3.1.0.tgz#a205383fea322b33b5ae3b18abee0dc2f356ee15" diff --git a/webapp/components/ContributionForm/ContributionForm.spec.js b/webapp/components/ContributionForm/ContributionForm.spec.js index 0644c1321..eb4e9cd5a 100644 --- a/webapp/components/ContributionForm/ContributionForm.spec.js +++ b/webapp/components/ContributionForm/ContributionForm.spec.js @@ -1,7 +1,6 @@ import { config, mount } from '@vue/test-utils' import ContributionForm from './ContributionForm.vue' -import Vue from 'vue' import Vuex from 'vuex' import PostMutations from '~/graphql/PostMutations.js' @@ -17,15 +16,7 @@ config.stubs['nuxt-link'] = '' config.stubs['v-popover'] = '' describe('ContributionForm.vue', () => { - let wrapper, - postTitleInput, - expectedParams, - cancelBtn, - mocks, - propsData, - categoryIds, - englishLanguage, - deutschLanguage + let wrapper, postTitleInput, expectedParams, cancelBtn, mocks, propsData const postTitle = 'this is a title for a post' const postTitleTooShort = 'xx' let postTitleTooLong = '' @@ -52,8 +43,6 @@ describe('ContributionForm.vue', () => { slug: 'this-is-a-title-for-a-post', content: postContent, contentExcerpt: postContent, - language: 'en', - categoryIds, }, }, }), @@ -109,10 +98,6 @@ describe('ContributionForm.vue', () => { postTitleInput = wrapper.find('.ds-input') postTitleInput.setValue(postTitle) await wrapper.vm.updateEditorContent(postContent) - englishLanguage = wrapper - .findAll('li') - .filter((language) => language.text() === 'English') - englishLanguage.trigger('click') }) it('title cannot be empty', async () => { @@ -147,7 +132,6 @@ describe('ContributionForm.vue', () => { variables: { title: postTitle, content: postContent, - language: 'en', id: null, image: null, }, @@ -155,29 +139,13 @@ describe('ContributionForm.vue', () => { postTitleInput = wrapper.find('.ds-input') postTitleInput.setValue(postTitle) await wrapper.vm.updateEditorContent(postContent) - englishLanguage = wrapper - .findAll('li') - .filter((language) => language.text() === 'English') - englishLanguage.trigger('click') - await Vue.nextTick() - await Vue.nextTick() }) - it('creates a post with valid title, content, and at least one category', async () => { + it('creates a post with valid title and content', async () => { await wrapper.find('form').trigger('submit') expect(mocks.$apollo.mutate).toHaveBeenCalledWith(expect.objectContaining(expectedParams)) }) - it('supports changing the language', async () => { - expectedParams.variables.language = 'de' - deutschLanguage = wrapper - .findAll('li') - .filter((language) => language.text() === 'Deutsch') - deutschLanguage.trigger('click') - wrapper.find('form').trigger('submit') - expect(mocks.$apollo.mutate).toHaveBeenCalledWith(expect.objectContaining(expectedParams)) - }) - it('supports adding a teaser image', async () => { expectedParams.variables.image = { aspectRatio: null, @@ -235,13 +203,6 @@ describe('ContributionForm.vue', () => { postTitleInput = wrapper.find('.ds-input') postTitleInput.setValue(postTitle) await wrapper.vm.updateEditorContent(postContent) - categoryIds = ['cat12'] - englishLanguage = wrapper - .findAll('li') - .filter((language) => language.text() === 'English') - englishLanguage.trigger('click') - await Vue.nextTick() - await Vue.nextTick() }) it('shows an error toaster when apollo mutation rejects', async () => { @@ -260,14 +221,7 @@ describe('ContributionForm.vue', () => { slug: 'dies-ist-ein-post', title: 'dies ist ein Post', content: 'auf Deutsch geschrieben', - language: 'de', image, - categories: [ - { - id: 'cat12', - name: 'Democracy & Politics', - }, - ], }, } wrapper = Wrapper() @@ -290,8 +244,6 @@ describe('ContributionForm.vue', () => { slug: 'this-is-a-title-for-a-post', content: postContent, contentExcerpt: postContent, - language: 'en', - categoryIds, }, }, }) @@ -301,7 +253,6 @@ describe('ContributionForm.vue', () => { variables: { title: propsData.contribution.title, content: propsData.contribution.content, - language: propsData.contribution.language, id: propsData.contribution.id, image: { sensitive: false, diff --git a/webapp/components/ContributionForm/ContributionForm.vue b/webapp/components/ContributionForm/ContributionForm.vue index 8db89173f..42ed2799e 100644 --- a/webapp/components/ContributionForm/ContributionForm.vue +++ b/webapp/components/ContributionForm/ContributionForm.vue @@ -50,17 +50,6 @@ {{ contentLength }} - - - -
{{ $t('actions.cancel') }} @@ -76,10 +65,8 @@