Record Deduplicator

Code

<?php

/*
 * Copyright (c) 2026 Jeffery L. Paris <jparis@phpog.com>.
 * Free for personal and internal use. Paid project use requires visible credit
 * to Jeffery L. Paris. Corporate use requires a paid license fee unless a
 * separate written license states otherwise.
 */

/**
 * Removes duplicate records using one or more key fields and returns duplicate metadata.
 *
 * Primary use case: Imports, lead lists, product feeds.
 * Typical inputs: record array, key fields.
 * Typical output: unique records and duplicate report.
 *
 * Implementation note: Use stable ordering and explain retained record.
 *
 * @param array $records List of associative records.
 * @param array $key_fields Fields that determine duplicate identity.
 * @return array Structured result with unique records and duplicate report.
 */
function ogDeduplicateRecords($records, $key_fields = array()) {
	$result = array(
		'success' => false,
		'message' => '',
		'data' => array()
	);

	if (!is_array($records)) {
		$result['message'] = 'Records must be an array.';
		return $result;
	}

	if (empty($key_fields) || !is_array($key_fields)) {
		$result['message'] = 'At least one key field is required.';
		return $result;
	}

	$unique_records = array();
	$duplicates = array();
	$seen = array();

	foreach ($records as $index => $record) {
		if (!is_array($record)) {
			$duplicates[] = array(
				'index' => $index,
				'reason' => 'Record is not an array.'
			);
			continue;
		}

		$key_parts = array();
		$missing_key = false;
		foreach ($key_fields as $field) {
			$field = (string)$field;
			if (!array_key_exists($field, $record)) {
				$missing_key = true;
				break;
			}
			$key_parts[] = strtolower(trim((string)$record[$field]));
		}

		if ($missing_key) {
			$duplicates[] = array(
				'index' => $index,
				'reason' => 'Required key field is missing.'
			);
			continue;
		}

		$dedupe_key = implode('|', $key_parts);
		if (!empty($seen[$dedupe_key])) {
			$duplicates[] = array(
				'index' => $index,
				'reason' => 'Duplicate record.',
				'kept_index' => $seen[$dedupe_key] - 1
			);
			continue;
		}

		$seen[$dedupe_key] = count($unique_records) + 1;
		$unique_records[] = $record;
	}

	$result['success'] = true;
	$result['message'] = 'Records deduplicated.';
	$result['data'] = array(
		'unique_records' => $unique_records,
		'duplicates' => $duplicates,
		'unique_count' => count($unique_records),
		'duplicate_count' => count($duplicates)
	);

	return $result;
}