diff --git a/backend/package.json b/backend/package.json
index f6b66b07e..786fe6641 100644
--- a/backend/package.json
+++ b/backend/package.json
@@ -69,6 +69,7 @@
"helmet": "~3.22.0",
"ioredis": "^4.16.1",
"jsonwebtoken": "~8.5.1",
+ "languagedetect": "^2.0.0",
"linkifyjs": "~2.1.8",
"lodash": "~4.17.14",
"merge-graphql-schemas": "^1.7.7",
diff --git a/backend/src/middleware/index.js b/backend/src/middleware/index.js
index 83b0104ec..0ad8cb1ae 100644
--- a/backend/src/middleware/index.js
+++ b/backend/src/middleware/index.js
@@ -14,6 +14,7 @@ import notifications from './notifications/notificationsMiddleware'
import hashtags from './hashtags/hashtagsMiddleware'
import email from './email/emailMiddleware'
import sentry from './sentryMiddleware'
+import languages from './languages/languages'
export default (schema) => {
const middlewares = {
@@ -30,6 +31,7 @@ export default (schema) => {
softDelete,
includedFields,
orderBy,
+ languages,
}
let order = [
@@ -39,6 +41,7 @@ export default (schema) => {
// 'activityPub', disabled temporarily
'validation',
'sluggify',
+ 'languages',
'excerpt',
'email',
'notifications',
diff --git a/backend/src/middleware/languages/languages.js b/backend/src/middleware/languages/languages.js
new file mode 100644
index 000000000..3cf760f31
--- /dev/null
+++ b/backend/src/middleware/languages/languages.js
@@ -0,0 +1,28 @@
+import LanguageDetect from 'languagedetect'
+import sanitizeHtml from 'sanitize-html'
+
+const removeHtmlTags = (input) => {
+ return sanitizeHtml(input, {
+ allowedTags: [],
+ allowedAttributes: {},
+ })
+}
+
+const setPostLanguage = (text) => {
+ const lngDetector = new LanguageDetect()
+ lngDetector.setLanguageType('iso2')
+ return lngDetector.detect(removeHtmlTags(text), 1)[0][0]
+}
+
+export default {
+ Mutation: {
+ CreatePost: async (resolve, root, args, context, info) => {
+ args.language = await setPostLanguage(args.content)
+ return resolve(root, args, context, info)
+ },
+ UpdatePost: async (resolve, root, args, context, info) => {
+ args.language = await setPostLanguage(args.content)
+ return resolve(root, args, context, info)
+ },
+ },
+}
diff --git a/backend/src/middleware/languages/languages.spec.js b/backend/src/middleware/languages/languages.spec.js
new file mode 100644
index 000000000..1448e4e4b
--- /dev/null
+++ b/backend/src/middleware/languages/languages.spec.js
@@ -0,0 +1,132 @@
+import Factory, { cleanDatabase } from '../../db/factories'
+import { gql } from '../../helpers/jest'
+import { getNeode, getDriver } from '../../db/neo4j'
+import createServer from '../../server'
+import { createTestClient } from 'apollo-server-testing'
+
+let mutate
+let authenticatedUser
+let variables
+
+const driver = getDriver()
+const neode = getNeode()
+
+beforeAll(async () => {
+ const { server } = createServer({
+ context: () => {
+ return {
+ driver,
+ neode,
+ user: authenticatedUser,
+ }
+ },
+ })
+ mutate = createTestClient(server).mutate
+})
+
+afterAll(async () => {
+ await cleanDatabase()
+})
+
+const createPostMutation = gql`
+ mutation($title: String!, $content: String!, $categoryIds: [ID]) {
+ CreatePost(title: $title, content: $content, categoryIds: $categoryIds) {
+ language
+ }
+ }
+`
+
+describe('languagesMiddleware', () => {
+ variables = {
+ title: 'Test post languages',
+ categoryIds: ['cat9'],
+ }
+
+ beforeAll(async () => {
+ await cleanDatabase()
+ const user = await Factory.build('user')
+ authenticatedUser = await user.toJson()
+ await Factory.build('category', {
+ id: 'cat9',
+ name: 'Democracy & Politics',
+ icon: 'university',
+ })
+ })
+
+ it('detects German', async () => {
+ variables = {
+ ...variables,
+ content: 'Jeder sollte vor seiner eigenen Tür kehren.',
+ }
+ await expect(
+ mutate({
+ mutation: createPostMutation,
+ variables,
+ }),
+ ).resolves.toMatchObject({
+ data: {
+ CreatePost: {
+ language: 'de',
+ },
+ },
+ })
+ })
+
+ it('detects English', async () => {
+ variables = {
+ ...variables,
+ content: 'A journey of a thousand miles begins with a single step.',
+ }
+ await expect(
+ mutate({
+ mutation: createPostMutation,
+ variables,
+ }),
+ ).resolves.toMatchObject({
+ data: {
+ CreatePost: {
+ language: 'en',
+ },
+ },
+ })
+ })
+
+ it('detects Spanish', async () => {
+ variables = {
+ ...variables,
+ content: 'A caballo regalado, no le mires el diente.',
+ }
+ await expect(
+ mutate({
+ mutation: createPostMutation,
+ variables,
+ }),
+ ).resolves.toMatchObject({
+ data: {
+ CreatePost: {
+ language: 'es',
+ },
+ },
+ })
+ })
+
+ it('detects German in between lots of html tags', async () => {
+ variables = {
+ ...variables,
+ content:
+ 'Jeder sollte vor seiner eigenen
Türkehren.', + } + await expect( + mutate({ + mutation: createPostMutation, + variables, + }), + ).resolves.toMatchObject({ + data: { + CreatePost: { + language: 'de', + }, + }, + }) + }) +}) diff --git a/backend/src/middleware/slugify/uniqueSlug.js b/backend/src/middleware/slugify/uniqueSlug.js index ca37cd562..7cfb89c19 100644 --- a/backend/src/middleware/slugify/uniqueSlug.js +++ b/backend/src/middleware/slugify/uniqueSlug.js @@ -2,6 +2,7 @@ import slugify from 'slug' export default async function uniqueSlug(string, isUnique) { const slug = slugify(string || 'anonymous', { lower: true, + multicharmap: { Ä: 'AE', ä: 'ae', Ö: 'OE', ö: 'oe', Ü: 'UE', ü: 'ue', ß: 'ss' }, }) if (await isUnique(slug)) return slug diff --git a/backend/src/middleware/slugify/uniqueSlug.spec.js b/backend/src/middleware/slugify/uniqueSlug.spec.js index ff14a56ef..d002eae03 100644 --- a/backend/src/middleware/slugify/uniqueSlug.spec.js +++ b/backend/src/middleware/slugify/uniqueSlug.spec.js @@ -18,4 +18,16 @@ describe('uniqueSlug', () => { const isUnique = jest.fn().mockResolvedValue(true) expect(uniqueSlug(string, isUnique)).resolves.toEqual('anonymous') }) + + it('Converts umlaut to a two letter equivalent', async () => { + const umlaut = 'ÄÖÜäöüß' + const isUnique = jest.fn().mockResolvedValue(true) + await expect(uniqueSlug(umlaut, isUnique)).resolves.toEqual('aeoeueaeoeuess') + }) + + it('Removes Spanish enya and diacritics', async () => { + const diacritics = 'áàéèíìóòúùñçÁÀÉÈÍÌÓÒÚÙÑÇ' + const isUnique = jest.fn().mockResolvedValue(true) + await expect(uniqueSlug(diacritics, isUnique)).resolves.toEqual('aaeeiioouuncaaeeiioouunc') + }) }) diff --git a/backend/src/schema/resolvers/posts.spec.js b/backend/src/schema/resolvers/posts.spec.js index b24383fba..f0c57b8fb 100644 --- a/backend/src/schema/resolvers/posts.spec.js +++ b/backend/src/schema/resolvers/posts.spec.js @@ -317,19 +317,6 @@ describe('CreatePost', () => { expected, ) }) - - describe('language', () => { - beforeEach(() => { - variables = { ...variables, language: 'es' } - }) - - it('allows a user to set the language of the post', async () => { - const expected = { data: { CreatePost: { language: 'es' } } } - await expect(mutate({ mutation: createPostMutation, variables })).resolves.toMatchObject( - expected, - ) - }) - }) }) }) diff --git a/backend/yarn.lock b/backend/yarn.lock index 0bbc62515..7d6558da0 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -6302,6 +6302,11 @@ knuth-shuffle-seeded@^1.0.6: dependencies: seed-random "~2.2.0" +languagedetect@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/languagedetect/-/languagedetect-2.0.0.tgz#4b8fa2b7593b2a3a02fb1100891041c53238936c" + integrity sha512-AZb/liiQ+6ZoTj4f1J0aE6OkzhCo8fyH+tuSaPfSo8YHCWLFJrdSixhtO2TYdIkjcDQNaR4RmGaV2A5FJklDMQ== + latest-version@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-3.1.0.tgz#a205383fea322b33b5ae3b18abee0dc2f356ee15" diff --git a/webapp/components/ContributionForm/ContributionForm.spec.js b/webapp/components/ContributionForm/ContributionForm.spec.js index 0644c1321..eb4e9cd5a 100644 --- a/webapp/components/ContributionForm/ContributionForm.spec.js +++ b/webapp/components/ContributionForm/ContributionForm.spec.js @@ -1,7 +1,6 @@ import { config, mount } from '@vue/test-utils' import ContributionForm from './ContributionForm.vue' -import Vue from 'vue' import Vuex from 'vuex' import PostMutations from '~/graphql/PostMutations.js' @@ -17,15 +16,7 @@ config.stubs['nuxt-link'] = '