libtorrent/test/test_checking.cpp

/*

Copyright (c) 2013-2022, Arvid Norberg
Copyright (c) 2017, Steven Siloti
Copyright (c) 2018, Alden Torres
Copyright (c) 2018, d-komarov
All rights reserved.

You may use, distribute and modify this code under the terms of the BSD license,
see LICENSE file.
*/

#include <sys/stat.h> // for chmod

#include "libtorrent/session.hpp"
#include "libtorrent/session_params.hpp"
#include "test.hpp"
#include "disk_io_test.hpp"
#include "settings.hpp"
#include "setup_transfer.hpp"
#include "test_utils.hpp"
#include "libtorrent/create_torrent.hpp"
#include "libtorrent/alert_types.hpp"
#include "libtorrent/torrent_info.hpp"
#include "libtorrent/torrent_status.hpp"
#include "libtorrent/hex.hpp" // to_hex
#include "libtorrent/aux_/path.hpp"
#include "libtorrent/aux_/open_mode.hpp"
#include "libtorrent/load_torrent.hpp"

namespace {

namespace
{
	bool is_checking(int const state)
	{
		return state == lt::torrent_status::checking_files
#if TORRENT_ABI_VERSION == 1
			|| state == lt::torrent_status::queued_for_checking
#endif
			|| state == lt::torrent_status::checking_resume_data;
	}
}

enum
{
	// make sure we don't accidentally require files to be writable just to
	// check their hashes
	read_only_files = 1,

	// make sure we detect corrupt files and mark the appropriate pieces
	// as not had
	corrupt_files = 2,

	incomplete_files = 4,

	// make the files not be there when starting up, move the files in place and
	// force-recheck. Make sure the stat cache is cleared and let us pick up the
	// new files
	force_recheck = 8,

	// files that are *bigger* than expected still considered OK. We don't
	// truncate files willy-nilly. Checking is a read-only operation.
	extended_files = 16,

	v2 = 32,

	single_file = 64,
};

void test_checking(int const flags, lt::disk_io_constructor_type disk_io)
{
	using namespace lt;

	std::printf("\n==== TEST CHECKING %s%s%s%s%s%s%s=====\n\n"
		, (flags & read_only_files) ? "read-only-files ":""
		, (flags & corrupt_files) ? "corrupt ":""
		, (flags & incomplete_files) ? "incomplete ":""
		, (flags & force_recheck) ? "force_recheck ":""
		, (flags & extended_files) ? "extended_files ":""
		, (flags & v2) ? "v2 ":""
		, (flags & single_file) ? "single_file ":"");

	error_code ec;
	create_directory("test_torrent_dir", ec);
	if (ec) fprintf(stdout, "ERROR: creating directory test_torrent_dir: (%d) %s\n"
		, ec.value(), ec.message().c_str());

	int const piece_size = (flags & single_file) ? 0x8000 : 0x4000;

	auto const file_sizes = (flags & single_file)
		? std::vector<int>{500000}
		: std::vector<int>{0, 5, 16 - 5, 16000, 17, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
		,1,1,1,1,1,1,13,65000,34,75,2,30,400,50000,73000,900,43000,400,4300,6, 4, 16384 * 100, 16384 * 200, 16384 * 100};

	auto fs = create_random_files("test_torrent_dir", file_sizes);

	lt::create_torrent t(std::move(fs), piece_size, (flags & v2) ? create_torrent::v2_only : create_torrent::v1_only);

	// calculate the hash for all pieces
	set_piece_hashes(t, ".", ec);
	if (ec) std::printf("ERROR: set_piece_hashes: (%d) %s\n"
		, ec.value(), ec.message().c_str());

	add_torrent_params atp = load_torrent_buffer(bencode(t.generate()));
	auto ti = atp.ti;
	TEST_CHECK(ti->is_valid());

	std::printf("generated torrent: %s test_torrent_dir\n"
		, aux::to_hex(ti->info_hashes().v1).c_str());

	// truncate every file in half
	if (flags & (incomplete_files | extended_files))
	{
		for (std::size_t i = 0; i < file_sizes.size(); ++i)
		{
			if ((i % 3) == 2) continue;
			char name[1024];
			std::snprintf(name, sizeof(name), "test%d", int(i));
			char dirname[200];
			std::snprintf(dirname, sizeof(dirname), "test_dir%d", int(i) / 5);
			std::string path = combine_path("test_torrent_dir", dirname);
			path = combine_path(path, name);

			std::int64_t const new_len = (flags & extended_files)
				? file_sizes[i] + 10
				: file_sizes[i] * static_cast<int>(i % 3) / 2;

			int const ret = ::truncate(path.c_str(), new_len);
			if (ret < 0)
			{
				std::printf("ERROR: truncating file \"%s\": (%d) %s\n"
					, path.c_str(), errno, strerror(errno));
			}
		}
	}

	// overwrite the files with new random data
	if (flags & corrupt_files)
	{
		std::printf("corrupt file test. overwriting files\n");
		// increase the size of some files. When they're read only that forces
		// the checker to open them in write-mode to truncate them
		std::vector<int> const file_sizes2
		{{ 0, 5, 16 - 5, 16001, 30, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
			,1,1,1,1,1,1,13,65000,34,75,2,30,400,500,23000,900,43000,400,4300,6, 4}};
		create_random_files("test_torrent_dir", (flags & single_file) ? file_sizes : file_sizes2);
	}

	// make the files read only
	if (flags & read_only_files)
	{
		std::printf("making files read-only\n");
		for (std::size_t i = 0; i < file_sizes.size(); ++i)
		{
			char name[1024];
			std::snprintf(name, sizeof(name), "test%d", int(i));
			char dirname[200];
			std::snprintf(dirname, sizeof(dirname), "test_dir%d", int(i) / 5);

			std::string path = combine_path("test_torrent_dir", dirname);
			path = combine_path(path, name);
			std::printf("   %s\n", path.c_str());

#ifdef TORRENT_WINDOWS
			SetFileAttributesA(path.c_str(), FILE_ATTRIBUTE_READONLY);
#else
			chmod(path.c_str(), S_IRUSR);
#endif
		}
	}

	if (flags & force_recheck)
	{
		remove_all("test_torrent_dir_tmp", ec);
		if (ec) std::printf("ERROR: removing \"test_torrent_dir_tmp\": (%d) %s\n"
			, ec.value(), ec.message().c_str());
		rename("test_torrent_dir", "test_torrent_dir_tmp", ec);
		if (ec) std::printf("ERROR: renaming dir \"test_torrent_dir\": (%d) %s\n"
			, ec.value(), ec.message().c_str());
	}

	lt::session_params sp(settings());
	sp.disk_io_constructor = disk_io;
	lt::session ses1(sp);

	add_torrent_params p = std::move(atp);
	p.save_path = ".";
	torrent_handle tor1 = ses1.add_torrent(p, ec);
	TEST_CHECK(!ec);

	if (flags & force_recheck)
	{
		// first make sure the session tries to check for the file and can't find
		// them
		lt::alert const* a = wait_for_alert(
			ses1, torrent_checked_alert::alert_type, "checking");
		TEST_CHECK(a);

		// now, move back the files and force-recheck. make sure we pick up the
		// files this time
		remove_all("test_torrent_dir", ec);
		if (ec) fprintf(stdout, "ERROR: removing \"test_torrent_dir\": (%d) %s\n"
			, ec.value(), ec.message().c_str());
		rename("test_torrent_dir_tmp", "test_torrent_dir", ec);
		if (ec) fprintf(stdout, "ERROR: renaming dir \"test_torrent_dir_tmp\": (%d) %s\n"
			, ec.value(), ec.message().c_str());
		tor1.force_recheck();
	}

	torrent_status st;
	for (int i = 0; i < 20; ++i)
	{
		print_alerts(ses1, "ses1");

		st = tor1.status();

		std::printf("%d %f %s\n", st.state, st.progress_ppm / 10000.0, st.errc.message().c_str());

		if (!is_checking(st.state) || st.errc) break;
		std::this_thread::sleep_for(lt::milliseconds(500));
	}

	if (flags & (incomplete_files | corrupt_files))
	{
		TEST_CHECK(!st.is_seeding);

		std::this_thread::sleep_for(lt::milliseconds(500));
		st = tor1.status();

		TEST_CHECK(!st.errc);
		if (st.errc)
			std::printf("error: %s\n", st.errc.message().c_str());
		std::vector<std::int64_t> const file_progress = tor1.file_progress();
		bool one_incomplete = false;
		file_storage const& fs1 = ti->layout();
		for (file_index_t i : fs1.file_range())
		{
			if (fs1.pad_file_at(i)) continue;
			std::printf("file: %d progress: %" PRId64 " / %" PRId64 "\n", static_cast<int>(i)
				, file_progress[std::size_t(static_cast<int>(i))], fs1.file_size(i));
			if (fs1.file_size(i) == file_progress[std::size_t(static_cast<int>(i))]) continue;
			one_incomplete = true;
		}
		TEST_CHECK(one_incomplete);
		TEST_CHECK(st.num_pieces < ti->num_pieces());
	}
	else
	{
		TEST_CHECK(st.num_pieces == ti->num_pieces());
		TEST_CHECK(st.is_seeding);
		if (st.errc)
			std::printf("error: %s\n", st.errc.message().c_str());
	}

	// make the files writable again
	if (flags & read_only_files)
	{
		for (std::size_t i = 0; i < file_sizes.size(); ++i)
		{
			char name[1024];
			std::snprintf(name, sizeof(name), "test%d", int(i));
			char dirname[200];
			std::snprintf(dirname, sizeof(dirname), "test_dir%d", int(i) / 5);

			std::string path = combine_path("test_torrent_dir", dirname);
			path = combine_path(path, name);
#ifdef TORRENT_WINDOWS
			SetFileAttributesA(path.c_str(), FILE_ATTRIBUTE_NORMAL);
#else
			chmod(path.c_str(), S_IRUSR | S_IWUSR);
#endif
		}
	}

	remove_all("test_torrent_dir", ec);
	if (ec) std::printf("ERROR: removing test_torrent_dir: (%d) %s\n"
		, ec.value(), ec.message().c_str());
}

} // anonymous namespace

TORRENT_TEST_DISK_IO(checking) { test_checking(0, disk_io); }

TORRENT_TEST_DISK_IO(read_only_corrupt) { test_checking(read_only_files | corrupt_files, disk_io); }

TORRENT_TEST_DISK_IO(read_only) { test_checking(read_only_files, disk_io); }

TORRENT_TEST_DISK_IO(incomplete) { test_checking(incomplete_files, disk_io); }

TORRENT_TEST_DISK_IO(extended) { test_checking(extended_files, disk_io); }

TORRENT_TEST_DISK_IO(corrupt) { test_checking(corrupt_files, disk_io); }

TORRENT_TEST_DISK_IO(force_recheck) { test_checking(force_recheck, disk_io); }

TORRENT_TEST_DISK_IO(checking_v2) { test_checking(v2, disk_io); }

TORRENT_TEST_DISK_IO(read_only_corrupt_v2)
{
	test_checking(read_only_files | corrupt_files | v2, disk_io);
}

TORRENT_TEST_DISK_IO(read_only_v2) { test_checking(read_only_files | v2, disk_io); }

TORRENT_TEST_DISK_IO(incomplete_v2) { test_checking(incomplete_files | v2, disk_io); }

TORRENT_TEST_DISK_IO(corrupt_v2) { test_checking(corrupt_files | v2, disk_io); }

TORRENT_TEST_DISK_IO(single_file_v2) { test_checking(v2 | single_file, disk_io); }

TORRENT_TEST_DISK_IO(single_file_corrupt_v2)
{
	test_checking(corrupt_files | v2 | single_file, disk_io);
}

TORRENT_TEST_DISK_IO(single_file_incomplete_v2)
{
	test_checking(incomplete_files | v2 | single_file, disk_io);
}

TORRENT_TEST_DISK_IO(force_recheck_v2) { test_checking(force_recheck | v2, disk_io); }

TORRENT_TEST_DISK_IO(discrete_checking)
{
	using namespace lt;
	printf("\n==== TEST CHECKING discrete =====\n\n");
	error_code ec;
	create_directory("test_torrent_dir", ec);
	if (ec) printf("ERROR: creating directory test_torrent_dir: (%d) %s\n", ec.value(), ec.message().c_str());

	int const megabyte = 0x100000;
	int const piece_size = 2 * megabyte;
	static std::array<int const, 2> const file_sizes{{ 9 * megabyte, 3 * megabyte }};

	auto fs = create_random_files("test_torrent_dir", file_sizes);
	TEST_EQUAL(fs.size(), 2);

	lt::create_torrent t(std::move(fs), piece_size);
	set_piece_hashes(t, ".", ec);
	if (ec) printf("ERROR: set_piece_hashes: (%d) %s\n", ec.value(), ec.message().c_str());

	std::vector<char> const buf = bencode(t.generate());
	add_torrent_params atp = load_torrent_buffer(buf);
	auto ti = atp.ti;
	printf("generated torrent: %s test_torrent_dir result: %s\n"
		, aux::to_hex(ti->info_hashes().v1.to_string()).c_str()
		, ec.message().c_str());

	TEST_CHECK(ti->is_valid());

	// we have two files, but there are also two padfiles now
	TEST_EQUAL(ti->num_files(), 4);

	{
		lt::session_params sp(settings());
		sp.disk_io_constructor = disk_io;
		session ses1(sp);
		add_torrent_params p = std::move(atp);
		p.file_priorities.resize(std::size_t(ti->num_files()));
		p.file_priorities[0] = 1_pri;
		p.save_path = ".";
		torrent_handle tor1 = ses1.add_torrent(p, ec);
		// change the priority of a file while checking and make sure it doesn't interrupt the checking.
		std::vector<download_priority_t> prio(std::size_t(ti->num_files()), 0_pri);
		prio[2] = 1_pri;
		tor1.prioritize_files(prio);
		TEST_CHECK(wait_for_alert(ses1, torrent_checked_alert::alert_type
			, "torrent checked", pop_alerts::pop_all, seconds(50)));
		TEST_CHECK(tor1.status({}).is_seeding);
	}
	remove_all("test_torrent_dir", ec);
	if (ec) fprintf(stdout, "ERROR: removing test_torrent_dir: (%d) %s\n", ec.value(), ec.message().c_str());
}

// When need_picker() re-creates the piece picker, make sure we preserve the
// file priorities that have been set. piece priorities will not be preserved
// since they're stored inside the piece picker.
TORRENT_TEST_DISK_IO(preserve_file_priorities)
{
	using namespace lt;

	// Two files, each an exact multiple of piece_size so no piece spans a file
	// boundary. v1_only avoids pad files. This gives us:
	//  file 0 -> pieces 0-1  (will be set to dont_download)
	//  file 1 -> pieces 2-3  (will be set to default_priority)
	static char const* const test_dir = "test_torrent_dir_prio_recheck";
	int const piece_size = 0x4000;
	static std::array<int const, 2> const file_sizes{{ 2 * piece_size, 2 * piece_size }};

	// create_random_files creates the directories and files on disk and
	// populates fs with their paths so set_piece_hashes can hash them.
	auto fs = create_random_files(test_dir, file_sizes);

	error_code ec;
	lt::create_torrent ct(std::move(fs), piece_size, create_torrent::v1_only);
	set_piece_hashes(ct, ".", ec);

	std::vector<char> buf;
	bencode(std::back_inserter(buf), ct.generate());
	add_torrent_params p = load_torrent_buffer(buf);
	TEST_CHECK(p.ti->is_valid());

	p.save_path = ".";
	p.file_priorities = { dont_download, default_priority };

	lt::session_params sp(settings());
	sp.disk_io_constructor = disk_io;
	session ses(sp);
	torrent_handle h = ses.add_torrent(p, ec);
	TEST_CHECK(!ec);

	// Both files are on disk, so the initial check finds all pieces valid.
	// Even the dont_download file's pieces are marked "have", which makes
	// is_seeding() true. maybe_done_flushing() then destroys the piece picker
	// and sets m_have_all = true.
	TEST_CHECK(wait_for_alert(ses, torrent_checked_alert::alert_type
		, "initial check", pop_alerts::pop_all, seconds(30)));
	TEST_CHECK(h.status().is_seeding);

	// Delete file 0 before the forced recheck so its pieces won't be found.
	// This keeps the torrent out of seeding state after the recheck,
	// leaving the picker alive for priority inspection.
	std::string const file0_path = combine_path(test_dir
		, combine_path("test_dir0", "test0"));
	remove(file0_path, ec);

	// force_recheck() is called after the piece picker has been destroyed.
	h.force_recheck();

	TEST_CHECK(wait_for_alert(ses, torrent_checked_alert::alert_type
		, "recheck after file deletion", pop_alerts::pop_all, seconds(30)));

	// File 0 is gone, so the torrent must not be seeding.
	TEST_CHECK(!h.status().is_seeding);

	// Piece priorities must reflect the original file priorities:
	//  pieces 0-1: dont_download (priority 0) - file 0 was dont_download
	//  pieces 2-3: default_priority (4)       - file 1 was default_priority
	auto const prios = h.get_piece_priorities();
	TEST_EQUAL(int(prios.size()), 4);
	TEST_EQUAL(prios[0], dont_download);
	TEST_EQUAL(prios[1], dont_download);
	TEST_EQUAL(prios[2], default_priority);
	TEST_EQUAL(prios[3], default_priority);

	remove_all(test_dir, ec);
}