Content Similarity Scorer

Code

<?php

/*
 * Copyright (c) 2026 Jeffery L. Paris <jparis@phpog.com>.
 * Free for personal and internal use. Paid project use requires visible credit
 * to Jeffery L. Paris. Corporate use requires a paid license fee unless a
 * separate written license states otherwise.
 */

/**
 * Scores similarity between two text records using token overlap and normalized terms.
 *
 * Primary use case: Duplicate content detection and related items.
 * Typical inputs: text A, text B, scoring options.
 * Typical output: score from 0 to 1.
 *
 * Implementation note: Use as heuristic; not legal plagiarism detection.
 *
 * @param string $first_text First text value to compare.
 * @param string $second_text Second text value to compare.
 * @param array $options Optional stop_words and min_token_length settings.
 * @return array Structured result data with score, token counts, and shared terms.
 */
function ogScoreContentSimilarity($first_text, $second_text, $options = array()) {
	$result = array(
		'success' => false,
		'message' => '',
		'data' => array()
	);

	if (!is_array($options)) {
		$options = array();
	}

	$first_text = trim(strip_tags((string)$first_text));
	$second_text = trim(strip_tags((string)$second_text));

	if (empty($first_text)) {
		$result['message'] = 'First text is empty.';
		return $result;
	}

	if (empty($second_text)) {
		$result['message'] = 'Second text is empty.';
		return $result;
	}

	$min_token_length = 3;
	if (!empty($options['min_token_length'])) {
		$min_token_length = (int)$options['min_token_length'];
	}
	if ($min_token_length < 1) {
		$min_token_length = 1;
	}

	$stop_words = array('the', 'and', 'for', 'with', 'this', 'that', 'from', 'you', 'your', 'are', 'was', 'were', 'has', 'have');
	if (!empty($options['stop_words']) && is_array($options['stop_words'])) {
		$stop_words = $options['stop_words'];
	}

	$first_words = preg_split('/[^a-z0-9]+/i', strtolower($first_text));
	$second_words = preg_split('/[^a-z0-9]+/i', strtolower($second_text));
	$first_tokens = array();
	$second_tokens = array();

	foreach ($first_words as $word) {
		$word = trim($word);
		if (strlen($word) >= $min_token_length && !in_array($word, $stop_words, true)) {
			$first_tokens[$word] = true;
		}
	}

	foreach ($second_words as $word) {
		$word = trim($word);
		if (strlen($word) >= $min_token_length && !in_array($word, $stop_words, true)) {
			$second_tokens[$word] = true;
		}
	}

	$first_keys = array_keys($first_tokens);
	$second_keys = array_keys($second_tokens);
	$shared = array_values(array_intersect($first_keys, $second_keys));
	$union = array_values(array_unique(array_merge($first_keys, $second_keys)));
	$score = 0;

	if (count($union) > 0) {
		$score = count($shared) / count($union);
	}

	$result['success'] = true;
	$result['message'] = 'Content similarity scored.';
	$result['data'] = array(
		'score' => $score,
		'score_percent' => round($score * 100, 2),
		'first_token_count' => count($first_keys),
		'second_token_count' => count($second_keys),
		'shared_token_count' => count($shared),
		'shared_terms' => $shared
	);

	return $result;
}