Easy To Read Pak Writer (Code)

[Baker]

Code: Select all

// Pak file structure
// 1. Header (12 bytes. "PACK" followed by offset to directory info and the length of that directory info)
// 2. File bytes one after another
// 3. Directory (56 chars for name, offset from start of file, length of data)

typedef struct
{
	char	name[56];
	int		filepos, filelen;
} packfile_t;

typedef struct
{
	char	id[4];
	int		dirofs;
	int		dirlen;
} packheader_t;



int File_Length_ByHandle (FILE* fileToCheck)
{
	int lengthOfFile;

	fseek (fileToCheck, 0, SEEK_END);	// Go to end
	lengthOfFile = ftell(fileToCheck);	// Read position, that is file length

	fseek (fileToCheck, 0, SEEK_SET);	// Set to start
	return lengthOfFile;
}

int File_Append_BinaryFile (FILE* writeHandle, const char* fileToRead)
{
	int		bytesWritten = 0;
	FILE*	readHandle = fopen (fileToRead, "rb");
	
	if (readHandle)
	{
		char	buf[4096];
		int		bufsize = sizeof(buf);

		int fileReadLength	= File_Length_ByHandle (readHandle);
		int remaining		= fileReadLength;



		printf ("Appending %i bytes from %s\n", fileReadLength, fileToRead);
	
		//CreatePath (dest);
	
		while (remaining)
		{
			int bytesThisPass = remaining < bufsize ? remaining : bufsize;

			fread  (buf, bytesThisPass, 1, readHandle);   // Read 4096 bytes in (or less if near EOF)
			fwrite (buf, bytesThisPass, 1, writeHandle);  // Write 4096 bytes out 

			remaining    -= bytesThisPass;
			bytesWritten += bytesThisPass;
		}

		fclose (readHandle);
	} 

	return bytesWritten;
}

int File_GetCursor (FILE* fileHandle)
{
	int position = ftell (fileHandle);

	return position;
}

void File_SetCursor_ToStart (FILE* fileHandle)
{
	fseek  (fileHandle, 0, SEEK_SET);
}

int PakFile_Create_From_List (char* outputPakName, text1024_t* listOfFiles, int numFiles)
{
	int				i;

	packheader_t	pakHeader;
	packfile_t		pakDirectory[4096];
	int				pakHeaderSize = sizeof(pakHeader);

	int				dataBytesWritten = 0, fileSize, markSpot;
	
	FILE*			pakHandle = fopen (outputPakName, "wb");
	
	if (!pakHandle)
	{
		printf ("Error opening file for write '%s'\n", outputPakName);
		return 0;
	}

	memset (&pakHeader, 0, sizeof(pakHeader));
	memset (pakDirectory, 0, sizeof(pakDirectory));

	// Write the header with invalid data.  Don't worry we will be updating this later.
	fwrite (&pakHeader, pakHeaderSize, 1, pakHandle);

	// Write the files
	for (i = 0; i < numFiles ; i ++)
	{
		char* curFile	= listOfFiles[i].text;
		
		markSpot = File_GetCursor (pakHandle); // LittleLong for Endian correctness

		dataBytesWritten += (fileSize = File_Append_BinaryFile (pakHandle, curFile) ); // Get file size, increment dataBytes

		pakDirectory[i].filepos = LittleLong (markSpot);
		pakDirectory[i].filelen = LittleLong (fileSize);
		StringLCopy (pakDirectory[i].name, String_SkipPath(curFile));
		
	}

	{
		// Store the offset then write the directory.  Data at pakDirectory with entry size of entry 0 times filecount
			
		int direntSize			= sizeof(pakDirectory[0]);
		int directoryStart		= File_GetCursor (pakHandle);
		int directoryByteLength = direntSize * numFiles;

		fwrite (pakDirectory, directoryByteLength, 1, pakHandle);
		
		// Construct final header
		pakHeader.id[0] = 'P';
		pakHeader.id[1] = 'A';
		pakHeader.id[2] = 'C';
		pakHeader.id[3] = 'K';
		pakHeader.dirofs = LittleLong (directoryStart);
		pakHeader.dirlen = LittleLong (directoryByteLength);

		// Go back to the beginning and rewrite the header

		File_SetCursor_ToStart (pakHandle);
		fwrite (&pakHeader, pakHeaderSize, 1, pakHandle); 
	}

	// close the file

	fclose (pakHandle);
	return 1;
}

"Comparing files pak_x.pak and pakscape_pak_x.pak
FC: no differences encountered"

Entire little utility source (just 1 file):

http://quake-1.com/docs/utils/pakMake.c.txt

[Baker]

Approaching completion of some code that will allow easy modification and manipulation of pak files. (Probably to be followed by WAD2/WAD3 ... not too much difference).

Kind of have a finalized "scheme" on how to use the code:

Looks like this ...

void* myPack = PakFile_Open ("c:/quake/mypak.pak", ARCHIVE_CREATE);

PakFile_Add_File_As (myPack, "c:/quake/player.mdl", "player.mdl");
PakFile_Add_File_As (myPack, "c:/quake/shambler.mdl", "shambler.mdl");
PakFile_Close (myPack);

Also have stuff like:

void* myPack = PakFile_Open ("c:/quake/mypak.pak", ARCHIVE_EDIT);
PakFile_Delete_File (myPack, "shambler.mdl");
PakFile_Add_File_As (myPack, "shambler2.mdl");
PakFile_Close (myPack);

Yeah the code to use that stuff really does look like that. PakFile_Open allocates a nice struct to store the filehandle, filename and everything transparently so passing "myPack" around handles stuff. PakFile_Close obviously frees that memory, closes the file and writes stuff.

Code: Select all

//**********************************************************************************************************\
//
//	archive_pak.c:	Pak files
//
//**********************************************************************************************************/

#include "project.h"

#define MAX_PAK_FILES_IS_4096	4096
#define MAX_PAK_FILENAME_IS_55	55

// Pak file structure
// 1. Header
// 2. File bytes one after another
// 3. Directory

typedef struct
{
	char		id[4];
	int			directoryOffset;
	int			directoryLength;
} packheader_t;

typedef struct
{
	char		name[56];
	int			filepos;
	int			filelen;
} packfile_t;



//	char			tempName[MAX_PATH_IS_1024];			// NO!  Append data at the end of the file.  Deleted data?  Do nothing!


typedef struct pak_access_s
{
	packheader_t	header;
	packfile_t		directory[MAX_PAK_FILES_IS_4096];

	char			fileName[MAX_PATH_IS_1024];	
	FILE*			fileHandle;
	int				fileMode;							// ARCHIVE_CREATE, ARCHIVE_EDIT, ARCHIVE_READ

	fbool			isDirty;							// To avoid re-saving a Pack_File_Edit where nothing of substance was done
	int				totalEntries;
	int				dataSegmentSize;
	
} pak_access_t;






//#define ARCHIVE_CREATE  1
//#define ARCHIVE_EDIT	2
//#define ARCHIVE_READ	3

#define OPEN_FILE_1 1
#define PAK_CHECK_2 2
#define DIR_SIZE_3  3


const char* accessCodes[] = 
{
	"wb", // ARCHIVE_CREATE 0
	"r+b", // ARCHIVE_EDIT 1
	"rb", // ARCHIVE_READ 2 
};

const char* pak_open_error[] =
{
	"Unknown",
	"Couldn't open file",
	"Not a pak file",
	"Too many files, 4096 max",
};

#define CHECK_MODE_1 1
#define CHECK_LENGTH_2 2
#define CHECK_NUM_FILES_3 3
#define CHECK_APPEND_4 4

const char* pak_add_error[] =
{
	"Unknown",
	"Pak mode is read-only, cannot write",
	"Filename for pak directory exceeds 55 characters",
	"Too many files in pak, 4096 max",
	"Couldn't append data from file to pak",
};


// Browse through in-memory representation
int PakFile_Find_File_Index (struct pak_access_s* myPack, const char* storedName)
{
	int i;
	
	for (i = 0; i < myPack->totalEntries; i++)
		if (StringMatch (storedName, myPack->directory[i].name))
			return i;
	
	return -1;
}

fbool PakFile_Delete_File (struct pak_access_s* myPack, const char* storeAsName)
{
	int i = PakFile_Find_File_Index (myPack, storeAsName);
	
	if (i == -1) return False; // Nothing to do.

	if (myPack->fileMode != ARCHIVE_EDIT)
	{
		printf ("Access mode is not EDIT and does not allow entry delete\n");
		return False; // Can't delete
	}
	
	myPack->directory[i].name[0] = '\0'; // Set name to zero length
	return True;

}

fbool PakFile_Add_File_As (struct pak_access_s* myPack, const char* diskFileName, const char* storeAsName)
{
	int i = myPack->totalEntries, stage, oldwritecursor = File_GetCursor (myPack->fileHandle);
	
	stage = CHECK_MODE_1;		if (myPack->fileMode == ARCHIVE_READ)				goto pak_add_fail;
	stage = CHECK_LENGTH_2;		if (strlen(storeAsName) > MAX_PAK_FILENAME_IS_55)	goto pak_add_fail;
	stage = CHECK_NUM_FILES_3;	if (myPack->totalEntries >= MAX_PAK_FILES_IS_4096)	goto pak_add_fail;

	myPack->directory[i].filepos = File_GetCursor		  (myPack->fileHandle);  // LittleLong
	myPack->directory[i].filelen = File_Append_BinaryFile (myPack->fileHandle, storeAsName); // This writes the file.   LittleLong

	stage = CHECK_APPEND_4;		if (myPack->directory[i].filelen == 0)				goto pak_add_fail;
	
	// Everything OK
	StringLCopy (myPack->directory[i].name, storeAsName);	// Finalize the name
	myPack->isDirty	= True;									// Edit file needs written
	myPack->dataSegmentSize += myPack->directory[i].filelen;// Increase data segment count
	myPack->totalEntries ++;								// Total entries + 1

	return True;
pak_add_fail:
	File_SetCursor	(myPack->fileHandle, oldwritecursor);	// Set the cursor back to where it was just in case
	printf ("Can't store '%s' as '%s': %s" , diskFileName, storeAsName, pak_add_error[stage]);
	return False;
}

struct pak_access_s* PakFile_Open (const char* myPakFileName, int myFileMode)
{
	pak_access_t*	myPack		= Memory_calloc (sizeof(pak_access_t), 1, "In-Memory Pack File Struct");
	const char*		accessCode = accessCodes[myFileMode];
	int				stage		= OPEN_FILE_1;

	if ( !(myPack->fileHandle = fopen (myPakFileName, accessCode) ) )  goto pak_fail; // Assign filehandle, but if NULL then error out

	StringLCopy (myPack->fileName, myPakFileName);
	myPack->fileMode = myFileMode;	// Edit.  Create, write, read handled differently

	if (myFileMode == ARCHIVE_CREATE)  // If Create just write a dummy header and get out ...
		File_Safe_Write (myPack->fileHandle, &myPack->header, sizeof(myPack->header) );
	else
	{	// ... ekse read header/directory, set cursor to start of directory (EDIT may write new data there)
		
		File_Safe_Read				(myPack->fileHandle, &myPack->header, sizeof(myPack->header) );
		stage = PAK_CHECK_2;		if (memcmp(myPack->header.id, "PACK", 4))							goto pak_fail;
		stage = DIR_SIZE_3;			if (myPack->header.directoryLength > sizeof(myPack->directory) )	goto pak_fail;
		File_SetCursor				(myPack->fileHandle, myPack->header.directoryOffset);						// Set cursor
		File_Safe_Read				(myPack->fileHandle, myPack->directory, myPack->header.directoryLength);	// Get directory
		File_SetCursor				(myPack->fileHandle, myPack->header.directoryOffset);						// Move cursor to beginning of directory
		myPack->totalEntries		= myPack->header.directoryLength / sizeof(myPack->directory[0]);
		myPack->dataSegmentSize		= myPack->header.directoryOffset - sizeof(myPack->header);
	}   
	return myPack;  // Everything ok.  Get out with struct calloc'd and file open ...

pak_fail:   

	printf ("'%s': %s", myPakFileName, pak_open_error[stage]);
	if (myPack->fileHandle)  fclose (myPack->fileHandle); // Close file if we had one open
	myPack = Memory_free(myPack);
	return NULL;

}

[taniwha]

Your code looks pretty good, but...

QuakeForge has had a "pak" utility for quite some time (early 2002: I believe I'd written one earlier, but I can't find it). It's usage is very similar to tar (main difference is the us of - before the commands), though I think it's a one-shot deal (you can't add files to an existing pak).

As a bonus, QF also has a "zpak" script that extracts the files from a pak, runs gzip over the files, then rebuilds the pak with the gzipped files. QF does fully transparent gzip decompression: it will look for (eg) start.bsp and start.bsp.gz, and irrespective of the extension, check the file for gzip compression and decompress as it reads (unless told otherwise). pak0.pak and pak1.pak use 23MB combined, and then with QF's ogg support and streamed sounds, the music takes 66MB

Before you embark on WAD2/WAD3, you might want to take a look at qf's wad tool (tools/wad). One bit of warning though, wad.c and pak.c are really just the command-line interface: the format handling code is in libs/util/wadfile.c and libs/util/pakfile.c. In general, anything "shared" between engine and tools is in libs (usually util, but some other dirs (eg, image, gamecode) are also shared).

[taniwha]

Ok, I decided to take a closer look at your code, and found a few things to point out.

Code: Select all

#define MAX_PAK_FILES_IS_4096   4096

While it's great you're using a macro instead of a magic number, putting that number into the macro is, in the long run, counter productive (and impossible (without munging) for floats). The biggest problem is now when you want to change the max, you have to change not only the value of the define, but the name of the define and all usages of it (or end up creating conflicting information: worse than magic numbers). (btw, qf has no limit on pak file size).

Code: Select all

"Too many files, 4096 max",

This one is a problem because it would be much better to use something like

Code: Select all

printf ("Too many files, %d max", MAX_PAK_FILES);

. One less place to edit. The problem is that you then have to use a switch to select the right error message (but then, this is safer if you forget to create a new message when you create a new error code: you can use default as a catch all error).

Also, while it's a bit of a PITA, it's much better to use fprintf (stderr, ...) for error messages.

I assume StringLCopy is your string buffer safety function. Good move

Still, overall, it seems to be good code And nice to see someone who's not afraid to use goto

[Baker]

While it's great you're using a macro instead of a magic number, putting that number into the macro is, in the long run, counter productive (and impossible (without munging) for floats). The biggest problem is now when you want to change the max, you have to change not only the value of the define, but the name of the define and all usages of it (or end up creating conflicting information: worse than magic numbers). (btw, qf has no limit on pak file size).

I agree with that in theory, but in practice a #define is a essentially a constant and if it were likely to change it wouldn't be a constant.

And if something merits such a large change as changing a constant, you know as well as me that just changing the #define isn't how simple that works 9 times out of 10.

And a mass project-wide search and replace isn't so hard to change a constant.

Code: Select all

"Too many files, 4096 max",

This one is a problem because it would be much better to use something like

Code: Select all

printf ("Too many files, %d max", MAX_PAK_FILES);

. One less place to edit.

Yes, I don't like that in my code. But was willing to accept a certain level of imperfection to print error messages generically by stuffing them in an array. As far as I know, there isn't a nice and clean way to use a macro in a string constant and didn't want to customize the print styles of error messages,

Still, overall, it seems to be good code And nice to see someone who's not afraid to use goto

Sometimes goto is the right tool for the job. If it reduces the complexity of the code and increases readability. btw... the instances of printf aren't likely to survive, but I haven't decided on a plan to use the same code for both a console application and, say, either an engine or GUI utility.

[Baker]

Your code looks pretty good, but...

QuakeForge has had a "pak" utility for quite some time (early 2002: I believe I'd written one earlier, but I can't find it). It's usage is very similar to tar (main difference is the us of - before the commands), though I think it's a one-shot deal (you can't add files to an existing pak).

As a bonus, QF also has a "zpak" script that extracts the files from a pak, runs gzip over the files, then rebuilds the pak with the gzipped files. QF does fully transparent gzip decompression: it will look for (eg) start.bsp and start.bsp.gz, and irrespective of the extension, check the file for gzip compression and decompress as it reads (unless told otherwise). pak0.pak and pak1.pak use 23MB combined, and then with QF's ogg support and streamed sounds, the music takes 66MB

Before you embark on WAD2/WAD3, you might want to take a look at qf's wad tool (tools/wad). One bit of warning though, wad.c and pak.c are really just the command-line interface: the format handling code is in libs/util/wadfile.c and libs/util/pakfile.c. In general, anything "shared" between engine and tools is in libs (usually util, but some other dirs (eg, image, gamecode) are also shared).

I bet I'll take a look at your gzip stuff for sure.

My WAD stuff is essentially done in its most basic form:

Code: Select all

//**********************************************************************************************************\
//
//	archive_wad.c:	Wad files
//
//**********************************************************************************************************/

#include "project.h"

#define MAX_WAD_TEXTURE_NAME_LENGTH_IS_15	15

#define isWAD3(myWad) (myWad->entryChar == 'C')

typedef struct
{ 
	byte		id[4];				// "WAD2", Name of the new WAD format
	int			numEntries;         // Number of entries
	int			directoryOffset;    // Position of WAD directory in file
} wadheader_t;

typedef struct
{ 
	int			offset;				// Position of the entry in WAD
	int			wadsize;            // Size of the entry in WAD file
	int			memsize;            // Size of the entry in memory
	char		type;               // type of entry, should be 'D' (0x44) MipTex ('C' for Half-Life
	char		unused[3];          // Never compressed, so why bother
	char		name[16];           // 1 to 16 characters, '\0'-padded
} wadentry_t;

#define MAX_WAD_TEXTURES_IS_4096 4096
typedef struct wad_access_s
{
	wadheader_t		header;
	wadentry_t		directory[MAX_WAD_TEXTURES_IS_4096];

	char			fileName[MAX_PATH_IS_1024];
	FILE*			fileHandle;

	char			entryChar;

	int				totalFiles;
	int				dataBytesWritten;

} wad_access_t;



static struct wad_access_s* WADFile_Build_Start (const char* outWADFileName)
{
	wad_access_t*	myWAD = Memory_calloc (sizeof(wad_access_t), 1, "In-Memory Pack File Struct");	

	// Populate it
	StringLCopy (myWAD->fileName, outWADFileName);

	myWAD->totalFiles			= 0;
	myWAD->dataBytesWritten		= 0;

	myWAD->fileHandle = fopen (myWAD->fileName, "wb");

	if (!myWAD->fileHandle)
	{
		// Deallocate and return NULL
		myWAD = Memory_free(myWAD);
		return NULL;
	}

	// Write the header with invalid data.  Don't worry we will be updating this later.
	File_Safe_Write (myWAD->fileHandle, &myWAD->header, sizeof(myWAD->header) );
		
	return myWAD;

}


struct wad_access_s* WAD2File_Build_Start (const char* outWADFileName)
{
	wad_access_t* myWAD = WADFile_Build_Start (outWADFileName);

	if (myWAD)
		myWAD->entryChar = 'D'; // WAD2 uses D

	return myWAD;
}

struct wad_access_s* WAD3File_Build_Start (const char* outWADFileName)
{
	wad_access_t* myWAD = WADFile_Build_Start (outWADFileName);

	if (myWAD)
		myWAD->entryChar = 'C'; // WAD3 uses C

	return myWAD;
}

extern byte quakePalette [768];
// Name options ... TopDir or Bare 
fbool WADFile_Build_Add_File (struct wad_access_s* myWAD, const char* newFileName)
{
	char	newTextureName[MAX_PATH_IS_1024];
	byte	pad[2] = {0, 1};
	int		i = myWAD->totalFiles;

	if (myWAD->totalFiles >= MAX_WAD_TEXTURES_IS_4096)
	{
		printf ("WAD2 directory full with %i files", myWAD->totalFiles);
		return False;
	}

	// Prepare texture name.  #water becomes *water
	StringLCopy (newTextureName, String_SkipPath(newFileName));
	String_Edit_StripExtension (newTextureName);
	
	if (newTextureName[0] = '#') newTextureName[0] = '*'; // Fix warp texture names

	if (strlen(newTextureName) > MAX_WAD_TEXTURE_NAME_LENGTH_IS_15)
	{
		printf ("File name is too long '%s' with %i chars (MAX %i)\n", newTextureName, strlen(newTextureName), MAX_WAD_TEXTURE_NAME_LENGTH_IS_15);
		return False;
	}
	
	myWAD->directory[i].offset = File_GetCursor (myWAD->fileHandle);  // LittleLong
	
	{
		int startingBytesWritten = myWAD->dataBytesWritten;
		int netsize;

		myWAD->dataBytesWritten    += File_Append_BinaryFile (myWAD->fileHandle, newFileName); // This writes the file.   LittleLong

		if (isWAD3(myWAD))
		{
			myWAD->dataBytesWritten    +=  File_Safe_Write (myWAD->fileHandle, pad,		     sizeof(pad));			// 2
			myWAD->dataBytesWritten    +=  File_Safe_Write (myWAD->fileHandle, quakePalette, sizeof(quakePalette));	// 768
			
			// 4 byte align if WAD3, bumping up wadsize, memsize and dataBytesWritten
			while (myWAD->dataBytesWritten & 3)
				myWAD->dataBytesWritten += File_Safe_Write (myWAD->fileHandle, pad,				1);

		}

		// wadsize is disk size, which is always memsize ... right?
		myWAD->directory[i].wadsize = myWAD->directory[i].memsize = netsize = myWAD->dataBytesWritten - startingBytesWritten;
	}

//	myWAD->directory[i].type	 = 'D'; // 0x44 MipTex   WAD2
//	myWAD->directory[i].type	 = 'C'; // 0x43 MipTex   WAD3

	myWAD->directory[i].type	 = myWAD->entryChar;

	StringLCopy (myWAD->directory[i].name, newTextureName);

	myWAD->totalFiles ++;

	return True;
}

struct wad_access_s* WADFile_Build_Finish (struct wad_access_s* myWAD)
{
	// Store the offset then write the directory.  Data at pakDirectory with entry size of entry 0 times filecount
	int directoryStart			= File_GetCursor (myWAD->fileHandle);	
	int directoryEntrySize		= sizeof(myWAD->directory[0]);
	int directoryByteLength		= directoryEntrySize * myWAD->totalFiles;

	File_Safe_Write (myWAD->fileHandle, &myWAD->directory, directoryByteLength);
	
	// Construct final header
	myWAD->header.id[0]				= 'W';
	myWAD->header.id[1]				= 'A';
	myWAD->header.id[2]				= 'D';
	myWAD->header.id[3]				= isWAD3(myWAD) ?  '3' : '2';
	myWAD->header.numEntries		= LittleLong (myWAD->totalFiles);
	myWAD->header.directoryOffset	= LittleLong (directoryStart);

	// Go back to the beginning and rewrite the header

	File_SetCursor_ToStart (myWAD->fileHandle);
	File_Safe_Write (myWAD->fileHandle, &myWAD->header, sizeof(myWAD->header) );

	// close the file
	fclose (myWAD->fileHandle);

	// Free the memory (this should effective return NULL, since Memory_free returns NULL)
	return Memory_free (myWAD);
}

[taniwha]

I bet I'll take a look at your gzip stuff for sure.

In that case, the places to look are libs/util/quakeio.c and quakefs.c (and the corresponsing headers in include/QF).

re goto: yeah, tool for the job. I once tried the "do { ... break; ... } while (0);" construct (libs/models/skin.c, btw), but it felt like using a filed off nail in a Robinson screw.

[Baker]

Maybe final form:

Supports reading, editing, creating pak files. Simple access. Alpha sorts directory entries. Has a "fast edit" mode (appends new data, just removes directory entries of deleted files.)

Should you care? Not really. I can't imagine anything really needing this. Except something odd I'm making to auto-convert and palettize pak and wad textures from separate folders and do it to separate WADs and PAKs based on the folder name and do it fast.

// Create example

void* myPack = PakFile_Open ("mypack.pak", ARCHIVE_CREATE);
PakFile_Add_File_Pathless (myPack, "shambler.mdl");
PakFile_Add_File_As (myPack, "c:/my_models/something_whatever/tree.mdl", "progs/tree.mdl");
PakFile_Close (myPack);

// Edit example

void* myPack = PakFile_Open ("mypack.pak", ARCHIVE_EDIT);
PakFile_Add_File_Pathless (myPack, "shambler.mdl");
PakFile_Add_File_As (myPack, "c:/my_models/something_whatever/tree.mdl", "progs/tree.mdl");
PakFile_Delete_File (myPack, "zombie.mdl");
PakFile_Close (myPack);

Code: Select all

//**********************************************************************************************************\
//
//  archive_types.h:    Archive
//
//**********************************************************************************************************/

#define ARCHIVE_CREATE      0   // Sequential write-only.  No read.  No modification of commits.  No skipping around.
#define ARCHIVE_READ        1   // Read only.
#define ARCHIVE_EDIT        2   // Comprehensive edit
#define ARCHIVE_FASTEDIT    3   // Fast and wasteful edit.  Does not recover space for deleted files.


// archive_pak.c

struct pak_access_s* PakFile_Open (const char* myPakFileName, int myFileMode);
struct pak_access_s* PakFile_Close (struct pak_access_s* myPack);

fbool                PakFile_Delete_File (struct pak_access_s* myPack, const char* storeAsName);
fbool                PakFile_Rename_File (struct pak_access_s* myPack, const char* storeAsName, const char* newName);
fbool                PakFile_Add_File_As (struct pak_access_s* myPack, const char* diskFileName, const char* storeAsName);
#define PakFile_Add_File_Pathless(myPack, myPathName) PakFile_Add_File_As (myPack, myPathName, String_SkipPath(myPathName) )

fbool PakFile_File_Info (const char* myPakFileName);

Code: Select all

//**********************************************************************************************************\
//
//  archive_pak.c:  Pak files
//
//**********************************************************************************************************/

#include "project.h"

#define MAX_PAK_FILES_IS_4096   4096
#define MAX_PAK_FILENAME_IS_55  55

// Pak file structure
// 1. Header
// 2. File bytes one after another
// 3. Directory

typedef struct
{
    char        id[4];
    int         directoryOffset;
    int         directoryLength;
} packheader_t;

typedef struct
{
    char        name[56];
    int         filepos;
    int         filelen;
} packentry_t;

typedef struct pak_access_s
{
    packheader_t    header;
    packentry_t     directory[MAX_PAK_FILES_IS_4096];

    char            fileName[MAX_PATH_IS_1024]; 
    FILE*           fileHandle;
    int             fileMode;                           // ARCHIVE_CREATE, ARCHIVE_EDIT, ARCHIVE_READ

    fbool           isDirty;                            // To avoid re-saving a Pack_File_Edit where nothing of substance was done
    int             totalEntries;
    int             dataSegmentSize;
    
} pak_access_t;


const char* accessCodes[] = 
{
    "wb",   // ARCHIVE_CREATE 0
    "rb",   // ARCHIVE_READ 1
    "r+b",  // ARCHIVE_EDIT 2
    "r+b",  // ARCHIVE_FASTEDIT 3
};

#define OPEN_FILE_1 1
#define PAK_CHECK_2 2
#define DIR_SIZE_3  3

const char* pak_open_error[] =
{
    "Unknown",
    "Couldn't open file",
    "Not a pak file",
    "Too many files, 4096 max",
};

#define CHECK_MODE_1 1
#define CHECK_LENGTH_2 2
#define CHECK_NUM_FILES_3 3
#define CHECK_APPEND_4 4

const char* pak_add_error[] =
{
    "Unknown",
    "Pak mode is read-only, cannot write",
    "Filename for pak directory exceeds 55 characters",
    "Too many files in pak, 4096 max",
    "Couldn't append data from file to pak",
};

//
// SUPPORTING FUNCTIONS
//

static fbool PakFile_Close_Directory_AlphaSort (pak_access_t* myPack)
{
    fbool   didWeSort = False;

    if (myPack->fileMode == ARCHIVE_READ || myPack->totalEntries == 0)
        return False; // Read can't write or crazy situation with no entries

    didWeSort = Sort_Memory_Blocks_By_Name ((char*)myPack->directory, sizeof(myPack->directory[0]), myPack->totalEntries, 0);
    
    printf ("Did we alpha sort directory? %s ...\n", didWeSort ? "Yes" : "No" );
    return didWeSort; // We might have sorted.  We might have not needed to.
}

static fbool PakFile_AlphaSort_Test (pak_access_t* myPack)
{
    fbool   didWeSort = False;

    if (myPack->totalEntries == 0)
        return False; // Read can't write or crazy situation with no entries

    didWeSort = Sort_Memory_Blocks_By_Name ((char*)myPack->directory, sizeof(myPack->directory[0]), myPack->totalEntries, 0);
    
    return didWeSort; // We might have sorted.  We might have not needed to.
}


// Look through the pack and see if there is empty space
static fbool PakFile_Close_Check_For_Empty_Space (pak_access_t* myPack)
{
    // Assumes cursor is at the right place.  Create, Edit, FastEdit this is always true.  Read?  Well, make it so.
    int     directoryStart      = File_GetCursor (myPack->fileHandle);  
    int     expectedDataSize    = directoryStart - sizeof(myPack->header);
    int     foundSize           = 0;
    fbool   isEmptySpace        = False;
    int     i;

    if (myPack->fileMode == ARCHIVE_READ && directoryStart != myPack->header.directoryOffset)
        printf ("Hey chief, the cursor into the file seems to be in wrong place");

    for (i = 0; i < myPack->totalEntries; i ++)
        foundSize += myPack->directory[i].filelen;

    if (foundSize != expectedDataSize)
        isEmptySpace = True;

//  printf ("Needs compressed? %s ...\n", isEmptySpace ? "Yes" : "No" );
    return isEmptySpace;
}

// Removes directory entries of deleted files
static fbool PakFile_Close_Directory_Fixup (pak_access_t* myPack)
{
    fbool didAnything = False;
    int newNumEntries = 0;
    int i, j;
    
    if (myPack->fileMode == ARCHIVE_CREATE || myPack->fileMode == ARCHIVE_READ)
        return False; // "Create" doesn't support delete and read doesn't write at all

    // Determine actual number of entries
    for (i = 0; i < myPack->totalEntries; i ++)
        if (myPack->directory[i].name[0])
            newNumEntries ++; 

    // Recover space moving entries
    for (i = 0; i < newNumEntries; i ++)
        if (!myPack->directory[i].name[0]) // Is blank entry?
            for (j = i + 1;  i < myPack->totalEntries; i ++) // // Find a non-blank entry and move it in there
                if (myPack->directory[i].name[0])
                {
                    // Relocate and erase
                    memcpy (&myPack->directory[i], &myPack->directory[j], sizeof(myPack->directory[i]) );
                    memset (&myPack->directory[j], 0,                     sizeof(myPack->directory[j]) );
                    didAnything = True;
                    break; // Get out
                }

    myPack->totalEntries = newNumEntries;

    printf ("Did we clean up deleted files? %s ...\n", didAnything ? "Yes" : "No" );
    return didAnything;
}


// Make a copy without the extra space (where deleted data lives)
static void PakFile_Close_Compress (pak_access_t* myPack)
{
    const char* tempFileName = "temp.tmp";
    FILE*       fout         = fopen (tempFileName, "wb"); // Write-only
    int         cursor       = 0;
    int         oldsize      = myPack->header.directoryOffset - sizeof(myPack->header);
    int         i, newsize, recoveredsize;

    printf ("Compressing ...\n");
    if (!fout)
    {
        printf ("Space recover: Couldn't open temp file for writing\n");  // Let's see if this happens?
        return; // This can't be a fatal error.  Otherwise we lose additions, deletions and such.
    }

    cursor += File_Safe_Write (fout, &myPack->header, sizeof(myPack->header) ); // Just to keep things easy
    
    for (i = 0; i < myPack->totalEntries; i ++)
    {
        File_Append_FileBlock (fout, myPack->fileHandle, myPack->directory[i].filepos, myPack->directory[i].filelen);
        myPack->directory[i].filepos = cursor; // Update the file position
        cursor += myPack->directory[i].filelen;
    }

    // Write temp file directory and re-write the temp file header after refreshing it.
    {   
        int directoryStart          = File_GetCursor (fout);    
        int directoryByteLength     = sizeof(myPack->directory[0]) * myPack->totalEntries;
        newsize                     = directoryStart - sizeof(myPack->header);
        recoveredsize               = oldsize - newsize;

        myPack->header.id[0]        = 'P';      // Construct final header
        myPack->header.id[1]        = 'A';
        myPack->header.id[2]        = 'C';
        myPack->header.id[3]        = 'K';
        myPack->header.directoryOffset = LittleLong(directoryStart);
        myPack->header.directoryLength = LittleLong(directoryByteLength);
        
        File_Safe_Write             (fout, &myPack->directory, directoryByteLength);  // Write directory
        File_SetCursor_ToStart      (fout);                                         // Return to beginning of file
        File_Safe_Write             (fout, &myPack->header, sizeof(myPack->header) ); // Write header
    }

    fclose (fout);                  // New file completed.
    fclose (myPack->fileHandle);    // Old pack data is stale.
    
    File_Delete (myPack->fileName);
    File_Rename (tempFileName, myPack->fileName);

    myPack->fileHandle = fopen  (myPack->fileName, "rb"); // Re-open file, but in read-mode.
    printf ("Compress pack old size %i, new size %i, recovered size %i:", oldsize, newsize, recoveredsize);
}

// Browse through in-memory representation
static int PakFile_Find_File_Index (pak_access_t* myPack, const char* storedName)
{
    int i;
    
    for (i = 0; i < myPack->totalEntries; i++)
        if (StringMatch (storedName, myPack->directory[i].name))
            return i;
    
    return -1;
}

//
// ACTUAL FUNCTIONS (EXTERNAL)
//
fbool PakFile_File_Info (const char* myPakFileName)
{
    if (File_Exists (myPakFileName) == False)
    {
        printf ("File '%s' does not exist\n", myPakFileName);
        return False;
    }

    do
    {
        int fileSize = File_Size (myPakFileName);
        pak_access_t* myPack = PakFile_Open (myPakFileName, ARCHIVE_READ);
        int i;
        int dataSegmentSize = 0;

        printf ( "\n-------------------------\n" );
        printf (    "File name:            %s\n", myPack->fileName);
        printf (    "File size on disk:    %i\n", fileSize);
        printf (    "------------------------\n" );
        printf (    "CONTENT                 \n" );
        printf (    "------------------------\n" );
        printf (    "Dir Offset:           %i\n", myPack->header.directoryOffset );
        printf (    "Dir Length:           %i\n", myPack->header.directoryLength );
        printf (    "Entry Length:         %i\n", (int)sizeof(myPack->directory[0]) );
        printf (    "Num Files (Calc):     %i\n", ( myPack->header.directoryLength / sizeof(myPack->directory[0] ) ) );
        printf (    "------------------------\n" );
        printf (    "SEGMENTS                \n" );
        printf (    "------------------------\n" );
        printf (    "Size of header:       %i\n", sizeof(myPack->header) );
        printf (    "Size of data segment: %i\n", (myPack->header.directoryOffset - sizeof(myPack->header)) );
        printf (    "Size of directory:    %i\n", fileSize - myPack->header.directoryOffset );
        printf (    "                      --\n");
        printf (    "TOTAL:                %i\n", sizeof(myPack->header) + (myPack->header.directoryOffset - sizeof(myPack->header)) + (fileSize - myPack->header.directoryOffset) );
        printf (    "MATCHES FILESIZE:     %s\n", (fileSize == sizeof(myPack->header) + (myPack->header.directoryOffset - sizeof(myPack->header)) + (fileSize - myPack->header.directoryOffset)) ? "YES" : "**NO**" );
        printf (    "------------------------\n");
        printf (    "Directory               \n");
        
        for (i = 0; i < myPack->totalEntries; i ++)
        {
            dataSegmentSize += myPack->directory[i].filelen;
            printf ("%i: %8i  %s  \n", i, myPack->directory[i].filelen, myPack->directory[i].name);
        }

        printf (    "                      --\n");
        printf (    "TOTAL COMPUTED        %i\n", dataSegmentSize );
#define IsEqualYesNo(x, y) ((x) == (y)) ? "Yes" : "No"
        printf (    "MATCHES EXPECTED:     %s\n", IsEqualYesNo(dataSegmentSize, myPack->header.directoryOffset - sizeof(myPack->header))  );
        printf (    "DIFF::                %i\n", (myPack->header.directoryOffset - sizeof(myPack->header) ) - dataSegmentSize  );
        printf (    "------------------------\n");
        File_SetCursor (myPack->fileHandle, myPack->header.directoryOffset);
        printf (    "Needs Compressed?  %i          \n", (int)PakFile_Close_Check_For_Empty_Space(myPack));

        PakFile_Close (myPack);

    } while (0);

    //

    return True;
}

struct pak_access_s* PakFile_Open (const char* myPakFileName, int myFileMode)
{
    pak_access_t*   myPack      = Memory_calloc (sizeof(pak_access_t), 1, "In-Memory Pack File Struct");
    const char*     accessCode = accessCodes[myFileMode];
    int             stage       = OPEN_FILE_1;

    if ( !(myPack->fileHandle = fopen (myPakFileName, accessCode) ) )  goto pak_open_fail; // Assign filehandle, but if NULL then error out

    StringLCopy (myPack->fileName, myPakFileName);
    myPack->fileMode = myFileMode;  // Edit.  Create, write, read handled differently

    if (myFileMode == ARCHIVE_CREATE)  // If Create just write a dummy header and get out ...
    {
        File_Safe_Write (myPack->fileHandle, &myPack->header, sizeof(myPack->header) );
        myPack->isDirty = True;
    }
    else
    {   // ... ekse read header/directory, set cursor to start of directory (EDIT may write new data there)
        
        File_Safe_Read              (myPack->fileHandle, &myPack->header, sizeof(myPack->header) );

        stage = PAK_CHECK_2;        if (memcmp(myPack->header.id, "PACK", 4))                           goto pak_open_fail;
        stage = DIR_SIZE_3;         if (myPack->header.directoryLength > sizeof(myPack->directory) )    goto pak_open_fail;

        File_SetCursor              (myPack->fileHandle, myPack->header.directoryOffset);                       // Set cursor
        File_Safe_Read              (myPack->fileHandle, myPack->directory, myPack->header.directoryLength);    // Get directory
        File_SetCursor              (myPack->fileHandle, myPack->header.directoryOffset);                       // Move cursor to beginning of directory

        myPack->totalEntries        = myPack->header.directoryLength / sizeof(myPack->directory[0]);
        myPack->dataSegmentSize     = myPack->header.directoryOffset - sizeof(myPack->header);
    }   
    return myPack;  // Everything ok.  Get out with struct calloc'd and file open ...

pak_open_fail:   

    printf ("'%s': %s", myPakFileName, pak_open_error[stage]);
    if (myPack->fileHandle)  fclose (myPack->fileHandle); // Close file if we had one open
    myPack = Memory_free(myPack);
    return NULL;
}


fbool PakFile_Add_File_As (struct pak_access_s* myPack, const char* diskFileName, const char* storeAsName)
{
    int i = myPack->totalEntries, stage, oldwritecursor = File_GetCursor (myPack->fileHandle);
    
    stage = CHECK_MODE_1;       if (myPack->fileMode == ARCHIVE_READ)               goto pak_add_fail;
    stage = CHECK_LENGTH_2;     if (strlen(storeAsName) > MAX_PAK_FILENAME_IS_55)   goto pak_add_fail;
    stage = CHECK_NUM_FILES_3;  if (myPack->totalEntries >= MAX_PAK_FILES_IS_4096)  goto pak_add_fail;

    myPack->directory[i].filepos = File_GetCursor         (myPack->fileHandle);  // LittleLong
    myPack->directory[i].filelen = File_Append_BinaryFile (myPack->fileHandle, storeAsName); // This writes the file.   LittleLong

    stage = CHECK_APPEND_4;     if (myPack->directory[i].filelen == 0)              goto pak_add_fail;
    
    // Everything OK
    StringLCopy (myPack->directory[i].name, storeAsName);   // Finalize the name
    myPack->isDirty = True;                                 // Edit file needs written
    myPack->dataSegmentSize += myPack->directory[i].filelen;// Increase data segment count
    myPack->totalEntries ++;                                // Total entries + 1

    return True;

pak_add_fail:

    File_SetCursor  (myPack->fileHandle, oldwritecursor);   // Set the cursor back to where it was just in case
    printf ("Can't store '%s' as '%s': %s" , diskFileName, storeAsName, pak_add_error[stage]);
    return False;
}
fbool PakFile_Rename_File (struct pak_access_s* myPack, const char* storeAsName, const char* newName)
{
    int i = PakFile_Find_File_Index (myPack, storeAsName);
    
    if (i == -1)
    {
        printf ("Couldn't rename '%s' because couldn't find it in pak\n", storeAsName);
        return False; // Couldn't find it so nothing to do.
    }

    if (myPack->fileMode != ARCHIVE_EDIT && myPack->fileMode != ARCHIVE_FASTEDIT)
    {
        printf ("Access mode is not EDIT and does not allow entry rename\n");
        return False; // Can't delete
    }
    
    if (strlen(newName) > MAX_PAK_FILENAME_IS_55)
    {
        printf ("New file name too long '%s' is %i characters (MAX %i)\n", newName, strlen(newName), MAX_PAK_FILENAME_IS_55);
        return False; // Rename 
    }

    StringLCopy (myPack->directory[i].name, newName);
    myPack->isDirty = True;
    return True;
}

fbool PakFile_Delete_File (struct pak_access_s* myPack, const char* storeAsName)
{
    int i = PakFile_Find_File_Index (myPack, storeAsName);
    
    if (i == -1) return False; // Nothing to do.

    if (myPack->fileMode != ARCHIVE_EDIT)
    {
        printf ("Access mode is not EDIT and does not allow entry delete\n");
        return False; // Can't delete
    }
    
    myPack->directory[i].name[0] = '\0'; // Set name to zero length

    myPack->isDirty = True;
    return True;

}

// File cursor must be at end of file
struct pak_access_s* PakFile_Close (struct pak_access_s* myPack)
{
    fbool directoryWasCompressed    = PakFile_Close_Directory_Fixup (myPack); 
    fbool directoryWasAlphaSorted   = PakFile_Close_Directory_AlphaSort (myPack);
    fbool pakNeedsCompressed        = PakFile_Close_Check_For_Empty_Space (myPack);

    if (myPack->isDirty == True) // Store the offset then write the directory.
    {
        int directoryStart          = File_GetCursor (myPack->fileHandle);  
        int directoryByteLength     = sizeof(myPack->directory[0]) * myPack->totalEntries;

        myPack->header.id[0]        = 'P';      // Construct final header
        myPack->header.id[1]        = 'A';
        myPack->header.id[2]        = 'C';
        myPack->header.id[3]        = 'K';
        myPack->header.directoryOffset = LittleLong(directoryStart);
        myPack->header.directoryLength = LittleLong(directoryByteLength);
        
        File_Safe_Write         (myPack->fileHandle, &myPack->directory, directoryByteLength);  // Write directory
        File_SetCursor_ToStart  (myPack->fileHandle);                                           // Return to beginning of file
        File_Safe_Write         (myPack->fileHandle, &myPack->header, sizeof(myPack->header) ); // Write header
    }

    if (myPack->fileMode == ARCHIVE_EDIT && pakNeedsCompressed)  // Create can't need it, read can't do it, fastedit doesn't want it
        PakFile_Close_Compress (myPack);    // Recover deleted space. Re-writes whole file.

    fclose (myPack->fileHandle);  // close the file

    // Free the memory (this should effective return NULL, since Memory_free returns NULL)
    return Memory_free (myPack);
}

Requires this --- possible a bit unusual -- sort function:

Code: Select all

fbool Sort_Memory_Blocks_By_Name (char* memoryBlock, size_t memBlockSize, size_t blockCount, size_t stringOffset) 
{
    fbool   anySwap = False;
    fbool   didSwap = True;
    char*   tmpBuf = Memory_calloc (memBlockSize, 1, "Sort temp buf");
    int     limit   = blockCount - 1;
    int     i;


    while (didSwap) 
    {
        didSwap = False;

        for (i = 0; i < limit; i++)  // Last record must be sorted
        {
            char* stringLocation  = &memoryBlock[(i + 0) * memBlockSize + stringOffset]; // Point into the buffer where the text is
            char* stringLocation2 = &memoryBlock[(i + 1) * memBlockSize + stringOffset]; // +1

            if (String_isLowerAlphabetically (stringLocation, stringLocation2) )
            {
                char* block1 = &memoryBlock[(i + 0) * memBlockSize];
                char* block2 = &memoryBlock[(i + 1) * memBlockSize];

                memcpy (tmpBuf, block1, memBlockSize);
                memcpy (block1, block2, memBlockSize);
                memcpy (block2, tmpBuf, memBlockSize);
                didSwap = anySwap = True;
            }
        }
        limit--;
    }

    tmpBuf = Memory_free (tmpBuf); // Release out temp buffer
    return anySwap; // If we touched anything, we return True otherwise we didn't change a thing
}

Is this boring as hell? Yes. That's just what you have to do sometimes though ...