kaizen/external/capstone/suite/cstest/src/test_run.c

// Copyright © 2024 Rot127 <unisono@quyllur.org>
// SPDX-License-Identifier: BSD-3

#include "test_run.h"
#include "test_case.h"
#include "test_mapping.h"
#include "../../../utils.h"
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include "cmocka.h"
#include <capstone/capstone.h>
#include <stdbool.h>
#include <stdio.h>

static TestRunResult get_test_run_result(TestRunStats *stats)
{
	if (stats->tc_total !=
	    stats->successful + stats->failed + stats->skipped) {
		fprintf(stderr,
			"[!] Inconsistent statistics: total != successful + failed + skipped\n");
		stats->errors++;
		return TEST_RUN_ERROR;
	}

	if (stats->errors != 0) {
		return TEST_RUN_ERROR;
	} else if (stats->failed != 0) {
		return TEST_RUN_FAILURE;
	}
	return TEST_RUN_SUCCESS;
}

/// Extract all test cases from the given test files.
static TestFile **parse_test_files(char **tf_paths, uint32_t path_count,
				   TestRunStats *stats)
{
	TestFile **files = NULL;
	stats->tc_total = 0;

	for (size_t i = 0; i < path_count; ++i) {
		TestFile *test_file_data = NULL;
		cyaml_err_t err = cyaml_load_file(
			tf_paths[i], &cyaml_config, &test_file_schema,
			(cyaml_data_t **)&test_file_data, NULL);

		if (err != CYAML_OK || !test_file_data) {
			fprintf(stderr, "[!] Failed to parse test file '%s'\n",
				tf_paths[i]);
			fprintf(stderr, "[!] Error: '%s'\n",
				!test_file_data && err == CYAML_OK ?
					"Empty file" :
					cyaml_strerror(err));
			stats->invalid_files++;
			stats->errors++;
			continue;
		}

		size_t k = stats->valid_test_files++;
		// Copy all test cases of a test file
		files = cs_mem_realloc(files, sizeof(TestFile *) *
						      stats->valid_test_files);

		files[k] = test_file_clone(test_file_data);
		assert(files[k]);
		stats->tc_total += files[k]->test_cases_count;
		files[k]->filename = strrchr(tf_paths[i], '/') ?
					     strdup(strrchr(tf_paths[i], '/')) :
					     strdup(tf_paths[i]);

		err = cyaml_free(&cyaml_config, &test_file_schema,
				 test_file_data, 0);
		if (err != CYAML_OK) {
			fprintf(stderr, "[!] Error: '%s'\n",
				cyaml_strerror(err));
			stats->errors++;
			continue;
		}
	}

	return files;
}

/// Parses the @input and saves the results in the other arguments.
static bool parse_input_options(const TestInput *input, cs_arch *arch,
				cs_mode *mode, cs_opt *opt_arr,
				size_t opt_arr_size, size_t *opt_set)
{
	assert(input && arch && mode && opt_arr);
	bool arch_found = false;
	const char *opt_str = input->arch;

	int val = enum_map_bin_search(test_arch_map, ARR_SIZE(test_arch_map),
				      opt_str, &arch_found);
	if (arch_found) {
		*arch = val;
	} else {
		fprintf(stderr,
			"[!] '%s' is not mapped to a capstone architecture.\n",
			input->arch);
		return false;
	}

	*mode = 0;
	size_t opt_idx = 0;
	char **options = input->options;
	for (size_t i = 0; i < input->options_count; ++i) {
		bool opt_found = false;
		opt_str = options[i];
		val = enum_map_bin_search(test_mode_map,
						   ARR_SIZE(test_mode_map),
						   opt_str, &opt_found);

		if (opt_found) {
			*mode |= val;
			continue;
		}

		// Might be an option descriptor
		for (size_t k = 0; k < ARR_SIZE(test_option_map); k++) {
			if (strings_match(opt_str, test_option_map[k].str)) {
				if (opt_idx >= opt_arr_size) {
					fprintf(stderr,
						"Too many options given in: '%s'. Maximum is: %" PRId64
						"\n",
						opt_str,
						(uint64_t)opt_arr_size);
					return false;
				}
				opt_arr[opt_idx++] = test_option_map[k].opt;
				opt_found = true;
				break;
			}
		}
		if (!opt_found) {
			fprintf(stderr, "[!] Option: '%s' not used\n", opt_str);
		}
	}
	*opt_set = opt_idx;
	return true;
}

/// Parses the options for cs_open/cs_option and initializes the handle.
/// Returns true for success and false otherwise.
static bool open_cs_handle(UnitTestState *ustate)
{
	cs_arch arch = 0;
	cs_mode mode = 0;
	cs_opt options[8] = { 0 };
	size_t options_set = 0;

	if (!parse_input_options(ustate->tcase->input, &arch, &mode, options, 8,
				 &options_set)) {
		char *tc_str = test_input_stringify(ustate->tcase->input, "");
		fprintf(stderr, "Could not parse options: %s\n", tc_str);
		cs_mem_free(tc_str);
		return false;
	}

	cs_err err = cs_open(arch, mode, &ustate->handle);
	if (err != CS_ERR_OK) {
		char *tc_str = test_input_stringify(ustate->tcase->input, "");
		fprintf(stderr,
			"[!] cs_open() failed with: '%s'. TestInput: %s\n",
			cs_strerror(err), tc_str);
		cs_mem_free(tc_str);
		return false;
	}

	// The bit mode must be set, otherwise the numbers are
	// not normalized correctly in the asm-test comparison step.
	if (arch == CS_ARCH_AARCH64 || mode & CS_MODE_64) {
		ustate->arch_bits = 64;
	} else if (mode & CS_MODE_16) {
		ustate->arch_bits = 16;
	} else {
		ustate->arch_bits = 32;
	}

	for (size_t i = 0; i < options_set; ++i) {
		err = cs_option(ustate->handle, options[i].type,
				options[i].val);
		if (err != CS_ERR_OK) {
			goto option_error;
		}
	}
	return true;

option_error: {
	char *tc_str = test_input_stringify(ustate->tcase->input, "");
	fprintf(stderr, "[!] cs_option() failed with: '%s'. TestInput: %s\n",
		cs_strerror(err), tc_str);
	cs_mem_free(tc_str);
	cs_close(&ustate->handle);
	return false;
}
}

static int cstest_unit_test_setup(void **state)
{
	assert(state);
	UnitTestState *ustate = *state;
	assert(ustate->tcase);
	if (!open_cs_handle(ustate)) {
		fail_msg("Failed to initialize Capstone with given options.");
		return -1;
	}
	return 0;
}

static int cstest_unit_test_teardown(void **state)
{
	if (!state) {
		return 0;
	}
	UnitTestState *ustate = *state;
	if (ustate->handle) {
		cs_err err = cs_close(&ustate->handle);
		if (err != CS_ERR_OK) {
			fail_msg("cs_close() failed with: '%s'.",
				 cs_strerror(err));
			return -1;
		}
	}
	return 0;
}

static void cstest_unit_test(void **state)
{
	assert(state);
	UnitTestState *ustate = *state;
	assert(ustate);
	assert(ustate->handle);
	assert(ustate->tcase);
	csh handle = ustate->handle;
	TestCase *tcase = ustate->tcase;

	cs_insn *insns = NULL;
	size_t insns_count = cs_disasm(handle, tcase->input->bytes,
				       tcase->input->bytes_count,
				       tcase->input->address, 0, &insns);
	test_expected_compare(&ustate->handle, tcase->expected, insns,
			      insns_count, ustate->arch_bits);
	ustate->decoded_insns += insns_count;
	cs_free(insns, insns_count);
}

static void eval_test_cases(TestFile **test_files, TestRunStats *stats)
{
	assert(test_files && stats);
	// CMocka's API doesn't allow to init a CMUnitTest with a partially initialized state
	// (which is later initialized in the test setup).
	// So we do it manually here.
	struct CMUnitTest *utest_table =
		cs_mem_calloc(sizeof(struct CMUnitTest),
			      stats->tc_total); // Number of test cases.

	char utest_id[128] = { 0 };

	size_t tci = 0;
	for (size_t i = 0; i < stats->valid_test_files; ++i) {
		TestCase **test_cases = test_files[i]->test_cases;
		const char *filename = test_files[i]->filename ?
					       test_files[i]->filename :
					       NULL;

		for (size_t k = 0; k < test_files[i]->test_cases_count;
		     ++k, ++tci) {
			cs_snprintf(utest_id, sizeof(utest_id),
				    "%s - TC #%" PRIx32 ": ", filename, k);
			if (test_cases[k]->skip) {
				char *tc_name = test_input_stringify(
					test_cases[k]->input, utest_id);
				fprintf(stderr, "SKIP: %s\nReason: %s\n",
					tc_name, test_cases[k]->skip_reason);
				cs_mem_free(tc_name);
				stats->skipped++;
				continue;
			}

			UnitTestState *ut_state =
				cs_mem_calloc(sizeof(UnitTestState), 1);
			ut_state->tcase = test_cases[k];
			utest_table[tci].name = test_input_stringify(
				ut_state->tcase->input, utest_id);
			utest_table[tci].initial_state = ut_state;
			utest_table[tci].setup_func = cstest_unit_test_setup;
			utest_table[tci].teardown_func =
				cstest_unit_test_teardown;
			utest_table[tci].test_func = cstest_unit_test;
		}
	}
	assert(tci == stats->tc_total);
	// Use private function here, because the API takes only constant tables.
	int failed_tests = _cmocka_run_group_tests(
		"All test cases", utest_table, stats->tc_total, NULL, NULL);
	assert(failed_tests >= 0 && "Faulty return value");

	for (size_t i = 0; i < stats->tc_total; ++i) {
		UnitTestState *ustate = utest_table[i].initial_state;
		if (!ustate) {
			// Skipped test case
			continue;
		}
		stats->decoded_insns += ustate->decoded_insns;
		cs_mem_free((char *)utest_table[i].name);
		cs_mem_free(utest_table[i].initial_state);
	}
	cs_mem_free(utest_table);
	stats->failed += failed_tests;
	stats->successful += stats->tc_total - failed_tests - stats->skipped;
}

/// Runs runs all valid tests in the given @test_files
/// and returns the result as well as statistics in @stats.
TestRunResult cstest_run_tests(char **test_file_paths, uint32_t path_count,
			       TestRunStats *stats)
{
	TestFile **files = parse_test_files(test_file_paths, path_count, stats);
	if (!files) {
		return get_test_run_result(stats);
	}
	eval_test_cases(files, stats);
	for (size_t i = 0; i < stats->valid_test_files; ++i) {
		test_file_free(files[i]);
	}
	cs_mem_free(files);

	return get_test_run_result(stats);
}