sanitize_test.go

package sanitize

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestFilterInvisibleCharacters(t *testing.T) {
	tests := []struct {
		name     string
		input    string
		expected string
	}{
		{
			name:     "empty string",
			input:    "",
			expected: "",
		},
		{
			name:     "normal text without invisible characters",
			input:    "Hello World",
			expected: "Hello World",
		},
		{
			name:     "text with zero width space",
			input:    "Hello\u200BWorld",
			expected: "HelloWorld",
		},
		{
			name:     "text with zero width non-joiner",
			input:    "Hello\u200CWorld",
			expected: "HelloWorld",
		},
		{
			name:     "text with left-to-right mark",
			input:    "Hello\u200EWorld",
			expected: "HelloWorld",
		},
		{
			name:     "text with right-to-left mark",
			input:    "Hello\u200FWorld",
			expected: "HelloWorld",
		},
		{
			name:     "text with soft hyphen",
			input:    "Hello\u00ADWorld",
			expected: "HelloWorld",
		},
		{
			name:     "text with zero width no-break space (BOM)",
			input:    "Hello\uFEFFWorld",
			expected: "HelloWorld",
		},
		{
			name:     "text with mongolian vowel separator",
			input:    "Hello\u180EWorld",
			expected: "HelloWorld",
		},
		{
			name:     "text with unicode tag character",
			input:    "Hello\U000E0001World",
			expected: "HelloWorld",
		},
		{
			name:     "text with unicode tag range characters",
			input:    "Hello\U000E0020World\U000E007FTest",
			expected: "HelloWorldTest",
		},
		{
			name:     "text with bidi control characters",
			input:    "Hello\u202AWorld\u202BTest\u202CEnd\u202DMore\u202EFinal",
			expected: "HelloWorldTestEndMoreFinal",
		},
		{
			name:     "text with bidi isolate characters",
			input:    "Hello\u2066World\u2067Test\u2068End\u2069Final",
			expected: "HelloWorldTestEndFinal",
		},
		{
			name:     "text with hidden modifier characters",
			input:    "Hello\u2060World\u2061Test\u2062End\u2063More\u2064Final",
			expected: "HelloWorldTestEndMoreFinal",
		},
		{
			name:     "multiple invisible characters mixed",
			input:    "Hello\u200B\u200C\u200E\u200F\u00AD\uFEFF\u180E\U000E0001World",
			expected: "HelloWorld",
		},
		{
			name:     "text with normal unicode characters (should be preserved)",
			input:    "Hello 世界 🌍 αβγ",
			expected: "Hello 世界 🌍 αβγ",
		},
		{
			name:     "invisible characters at start and end",
			input:    "\u200BHello World\u200C",
			expected: "Hello World",
		},
		{
			name:     "only invisible characters",
			input:    "\u200B\u200C\u200E\u200F",
			expected: "",
		},
		{
			name:     "real-world example with title",
			input:    "Fix\u200B bug\u00AD in\u202A authentication\u202C",
			expected: "Fix bug in authentication",
		},
		{
			name:     "issue body with mixed content",
			input:    "This is a\u200B bug report.\n\nSteps to reproduce:\u200C\n1. Do this\u200E\n2. Do that\u200F",
			expected: "This is a bug report.\n\nSteps to reproduce:\n1. Do this\n2. Do that",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := FilterInvisibleCharacters(tt.input)
			assert.Equal(t, tt.expected, result)
		})
	}
}

func TestShouldRemoveRune(t *testing.T) {
	tests := []struct {
		name     string
		rune     rune
		expected bool
	}{
		// Individual characters that should be removed
		{name: "zero width space", rune: 0x200B, expected: true},
		{name: "zero width non-joiner", rune: 0x200C, expected: true},
		{name: "left-to-right mark", rune: 0x200E, expected: true},
		{name: "right-to-left mark", rune: 0x200F, expected: true},
		{name: "soft hyphen", rune: 0x00AD, expected: true},
		{name: "zero width no-break space", rune: 0xFEFF, expected: true},
		{name: "mongolian vowel separator", rune: 0x180E, expected: true},
		{name: "unicode tag", rune: 0xE0001, expected: true},

		// Range tests - Unicode tags: U+E0020–U+E007F
		{name: "unicode tag range start", rune: 0xE0020, expected: true},
		{name: "unicode tag range middle", rune: 0xE0050, expected: true},
		{name: "unicode tag range end", rune: 0xE007F, expected: true},
		{name: "before unicode tag range", rune: 0xE001F, expected: false},
		{name: "after unicode tag range", rune: 0xE0080, expected: false},

		// Range tests - BiDi controls: U+202A–U+202E
		{name: "bidi control range start", rune: 0x202A, expected: true},
		{name: "bidi control range middle", rune: 0x202C, expected: true},
		{name: "bidi control range end", rune: 0x202E, expected: true},
		{name: "before bidi control range", rune: 0x2029, expected: false},
		{name: "after bidi control range", rune: 0x202F, expected: false},

		// Range tests - BiDi isolates: U+2066–U+2069
		{name: "bidi isolate range start", rune: 0x2066, expected: true},
		{name: "bidi isolate range middle", rune: 0x2067, expected: true},
		{name: "bidi isolate range end", rune: 0x2069, expected: true},
		{name: "before bidi isolate range", rune: 0x2065, expected: false},
		{name: "after bidi isolate range", rune: 0x206A, expected: false},

		// Range tests - Hidden modifiers: U+2060–U+2064
		{name: "hidden modifier range start", rune: 0x2060, expected: true},
		{name: "hidden modifier range middle", rune: 0x2062, expected: true},
		{name: "hidden modifier range end", rune: 0x2064, expected: true},
		{name: "before hidden modifier range", rune: 0x205F, expected: false},
		{name: "after hidden modifier range", rune: 0x2065, expected: false},

		// Characters that should NOT be removed
		{name: "regular ascii letter", rune: 'A', expected: false},
		{name: "regular ascii digit", rune: '1', expected: false},
		{name: "regular ascii space", rune: ' ', expected: false},
		{name: "newline", rune: '\n', expected: false},
		{name: "tab", rune: '\t', expected: false},
		{name: "unicode letter", rune: '世', expected: false},
		{name: "emoji", rune: '🌍', expected: false},
		{name: "greek letter", rune: 'α', expected: false},
		{name: "punctuation", rune: '.', expected: false},
		{name: "hyphen (normal)", rune: '-', expected: false},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := shouldRemoveRune(tt.rune)
			assert.Equal(t, tt.expected, result, "rune: U+%04X (%c)", tt.rune, tt.rune)
		})
	}
}