Improve deobfuscation (#469)

- improved deobfuscation:
  - enhanced parsing of the subject field in NZB files, along with improved validation of downloaded article filenames 
  - added a check for filename obfuscation after unpacking; excessively obfuscated filenames are renamed to the NZB filename if the NZB filename is not also obfuscated
- new options:
  - "RenameAfterUnpack" - enables/disables renaming of extracted and obfuscated files using the NZB filename
  - "RenameIgnoreExt" - a comma separated list of file extensions to ignore when renaming files after unpacking
- new value for field "Status" in API-method "listgroups": "POST_UNPACK_RENAMING"

daemon/main/Options.cpp

@@ -3,6 +3,7 @@
  *
  *  Copyright (C) 2004 Sven Henkel <[email protected]>
  *  Copyright (C) 2007-2019 Andrey Prygunkov <[email protected]>
+ *  Copyright (C) 2024 Denis <[email protected]>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -93,6 +94,8 @@ static const char* OPTION_PARSCAN				= "ParScan";
 static const char* OPTION_PARQUICK				= "ParQuick";
 static const char* OPTION_POSTSTRATEGY			= "PostStrategy";
 static const char* OPTION_FILENAMING			= "FileNaming";
+static const char* OPTION_RENAMEAFTERUNPACK	    = "RenameAfterUnpack";
+static const char* OPTION_RENAMEIGNOREEXT       = "RenameIgnoreExt";
 static const char* OPTION_PARRENAME				= "ParRename";
 static const char* OPTION_PARBUFFER				= "ParBuffer";
 static const char* OPTION_PARTHREADS			= "ParThreads";
@@ -480,6 +483,8 @@ void Options::InitDefaults()
 	SetOption(OPTION_PARQUICK, "yes");
 	SetOption(OPTION_POSTSTRATEGY, "sequential");
 	SetOption(OPTION_FILENAMING, "article");
+	SetOption(OPTION_RENAMEAFTERUNPACK, "yes");
+	SetOption(OPTION_RENAMEIGNOREEXT, ".zip, .7z, .rar, .par2");
 	SetOption(OPTION_PARRENAME, "yes");
 	SetOption(OPTION_PARBUFFER, "16");
 	SetOption(OPTION_PARTHREADS, "0");
@@ -703,6 +708,7 @@ void Options::InitOptions()
 	m_parIgnoreExt			= GetOption(OPTION_PARIGNOREEXT);
 	m_unpackIgnoreExt		= GetOption(OPTION_UNPACKIGNOREEXT);
 	m_shellOverride			= GetOption(OPTION_SHELLOVERRIDE);
+	m_renameIgnoreExt 	    = GetOption(OPTION_RENAMEIGNOREEXT);
 
 	m_downloadRate			= ParseIntValue(OPTION_DOWNLOADRATE, 10) * 1024;
 	m_articleTimeout		= ParseIntValue(OPTION_ARTICLETIMEOUT, 10);
@@ -773,6 +779,7 @@ void Options::InitOptions()
 	m_urlForce				= (bool)ParseEnumValue(OPTION_URLFORCE, BoolCount, BoolNames, BoolValues);
 	m_certCheck				= (bool)ParseEnumValue(OPTION_CERTCHECK, BoolCount, BoolNames, BoolValues);
 	m_reorderFiles			= (bool)ParseEnumValue(OPTION_REORDERFILES, BoolCount, BoolNames, BoolValues);
+	m_renameAfterUnpack     = (bool)ParseEnumValue(OPTION_RENAMEAFTERUNPACK, BoolCount, BoolNames, BoolValues);
 
 	const char* OutputModeNames[] = { "loggable", "logable", "log", "colored", "color", "ncurses", "curses" };
 	const int OutputModeValues[] = { omLoggable, omLoggable, omLoggable, omColored, omColored, omNCurses, omNCurses };

daemon/main/Options.h

@@ -3,6 +3,7 @@
  *
  *  Copyright (C) 2004 Sven Henkel <[email protected]>
  *  Copyright (C) 2007-2019 Andrey Prygunkov <[email protected]>
+ *  Copyright (C) 2024 Denis <[email protected]>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -294,6 +295,8 @@ class Options
 	bool GetUnpackPauseQueue() { return m_unpackPauseQueue; }
 	const char* GetExtCleanupDisk() { return m_extCleanupDisk; }
 	const char* GetParIgnoreExt() { return m_parIgnoreExt; }
+	const char* GetRenameIgnoreExt() { return m_renameIgnoreExt; }
+	bool GetRenameAfterUnpack() { return m_renameAfterUnpack; }
 	const char* GetUnpackIgnoreExt() { return m_unpackIgnoreExt; }
 	int GetFeedHistory() { return m_feedHistory; }
 	bool GetUrlForce() { return m_urlForce; }
@@ -444,6 +447,8 @@ class Options
 	int m_dailyQuota = 0;
 	bool m_reorderFiles = false;
 	EFileNaming m_fileNaming = nfArticle;
+	bool m_renameAfterUnpack = true;
+	CString m_renameIgnoreExt;
 	int m_downloadRate = 0;
 
 	// Application mode

daemon/postprocess/PostUnpackRenamer.cpp

@@ -0,0 +1,144 @@
+/*
+ *  This file is part of nzbget. See <https://nzbget.com>.
+ *
+ *  Copyright (C) 2024 Denis <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "nzbget.h"
+
+#include "PostUnpackRenamer.h"
+#include "FileSystem.h"
+#include "Deobfuscation.h"
+
+namespace PostUnpackRenamer
+{
+	void Controller::StartJob(PostInfo* postInfo)
+	{
+		Controller* controller = new (std::nothrow) Controller();
+
+		if (!controller)
+		{
+			error("Failed to allocate memory for PostUnpackRenamer::Controller");
+			return;
+		}
+
+		controller->m_postInfo = postInfo;
+		controller->SetAutoDestroy(false);
+
+		postInfo->SetPostThread(controller);
+
+		controller->Start();
+	}
+
+	void Controller::Run()
+	{
+		{
+			GuardedDownloadQueue guard = DownloadQueue::Guard();
+
+			m_name = m_postInfo->GetNzbInfo()->GetName();
+			m_dstDir = m_postInfo->GetNzbInfo()->GetDestDir();
+		}
+
+		std::string infoName = "Post-unpack renaming for " + m_name;
+		SetInfoName(infoName.c_str());
+
+		if (Deobfuscation::IsExcessivelyObfuscated(m_name))
+		{
+			PrintMessage(Message::mkWarning,
+				"Skipping Post-unpack renaming. NZB filename %s is excessively obfuscated which makes renaming unreliable.",
+				m_name.c_str()
+			);
+			m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus(
+				NzbInfo::PostUnpackRenamingStatus::Skipped
+			);
+			m_postInfo->SetWorking(false);
+			return;
+		}
+
+		bool ok = RenameFiles(m_dstDir, m_name);
+
+		GuardedDownloadQueue guard = DownloadQueue::Guard();
+		if (ok)
+		{
+			PrintMessage(Message::mkInfo, "%s successful", infoName.c_str());
+			m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus(
+				NzbInfo::PostUnpackRenamingStatus::Success
+			);
+		}
+		else
+		{
+			PrintMessage(Message::mkError, "%s failed", infoName.c_str());
+			m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus(
+				NzbInfo::PostUnpackRenamingStatus::Failure
+			);
+		}
+
+		m_postInfo->SetWorking(false);
+	}
+
+	bool Controller::RenameFiles(const std::string& dir, const std::string& newName)
+	{
+		DirBrowser dirBrowser(dir.c_str());
+		while (const char* fileOrDir = dirBrowser.Next())
+		{
+			std::string srcFileOrDir = dir + PATH_SEPARATOR + fileOrDir;
+
+			if (FileSystem::DirectoryExists(srcFileOrDir.c_str()))
+			{
+				RenameFiles(srcFileOrDir, newName);
+				continue;
+			}
+
+			if (!Deobfuscation::IsExcessivelyObfuscated(fileOrDir))
+			{
+				PrintMessage(Message::mkInfo,
+					"Filename %s is not excessively obfuscated, no renaming needed.",
+					fileOrDir
+				);
+				continue;
+			}
+
+			std::string dstFile = dir + PATH_SEPARATOR + newName;
+			dstFile += FileSystem::GetFileExtension(srcFileOrDir).value_or("");
+
+			if (Util::MatchFileExt(dstFile.c_str(), g_Options->GetRenameIgnoreExt(), ","))
+			{
+				continue;
+			}
+
+			if (FileSystem::MoveFile(srcFileOrDir.c_str(), dstFile.c_str()))
+			{
+				PrintMessage(Message::mkInfo, "%s renamed to %s", srcFileOrDir.c_str(), dstFile.c_str());
+			}
+			else
+			{
+				PrintMessage(Message::mkError,
+					"Could not rename file %s to %s: %s",
+					srcFileOrDir.c_str(),
+					dstFile.c_str(),
+					*FileSystem::GetLastErrorMessage()
+				);
+			}
+		}
+
+		return true;
+	}
+
+	void Controller::AddMessage(Message::EKind kind, const char* text)
+	{
+		m_postInfo->GetNzbInfo()->AddMessage(kind, text);
+	}
+}

daemon/postprocess/PostUnpackRenamer.h

@@ -0,0 +1,48 @@
+/*
+ *  This file is part of nzbget. See <https://nzbget.com>.
+ *
+ *  Copyright (C) 2024 Denis <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef POST_UNPACK_H
+#define POST_UNPACK_H
+
+#include <string>
+#include "Thread.h"
+#include "ScriptController.h"
+#include "DownloadInfo.h"
+
+namespace PostUnpackRenamer
+{
+	class Controller final : public Thread, public ScriptController
+	{
+	public:
+		void Run() override;
+		static void StartJob(PostInfo* postInfo);
+
+	protected:
+		void AddMessage(Message::EKind kind, const char* text) override;
+
+	private:
+		PostInfo* m_postInfo;
+		std::string m_name;
+		std::string m_dstDir;
+		bool RenameFiles(const std::string& dir, const std::string& nameToRename);
+	};
+}
+
+#endif

daemon/postprocess/PrePostProcessor.cpp

@@ -37,6 +37,7 @@
 #include "QueueScript.h"
 #include "ParParser.h"
 #include "DirectUnpack.h"
+#include "PostUnpackRenamer.h"
 
 PrePostProcessor::PrePostProcessor()
 {
@@ -844,6 +845,17 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 		unpack = false;
 	}
 
+	bool postUnpackRenaming = g_Options->GetRenameAfterUnpack() &&
+		nzbInfo->GetPostUnpackRenamingStatus() == NzbInfo::PostUnpackRenamingStatus::None &&
+		nzbInfo->GetDestDir() &&
+		nzbInfo->GetName() &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usFailure &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usSpace &&
+		nzbInfo->GetUnpackStatus() != NzbInfo::usPassword &&
+		nzbInfo->GetParStatus() != NzbInfo::psFailure &&
+		nzbInfo->GetParStatus() != NzbInfo::psManual &&
+		nzbInfo->GetMoveStatus() == NzbInfo::msSuccess;
+
 	if (unpack)
 	{
 		EnterStage(downloadQueue, postInfo, PostInfo::ptUnpacking);
@@ -859,6 +871,11 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
 		EnterStage(downloadQueue, postInfo, PostInfo::ptMoving);
 		MoveController::StartJob(postInfo);
 	}
+	else if (postUnpackRenaming)
+	{
+		EnterStage(downloadQueue, postInfo, PostInfo::ptPostUnpackRenaming);
+		PostUnpackRenamer::Controller::StartJob(postInfo);
+	}
 	else
 	{
 		EnterStage(downloadQueue, postInfo, PostInfo::ptExecutingScript);