#This script takes a newline delimited file list from STDIN for md5 hashing
#This script requires the `md5sum`, `pv`, `paste`, `bc`, and 'numfmt' commands

#The output of the md5s are stored in the file specified by the first parameter
#The format for each md5 hash to the output file is "$FileName\t$Hash\n"

#File sizes are always output in megabytes with 3 decimal places
#While calculating the hashes the script keeps the user informed of the progress of both the current file and all the files as follows:
#1) "Hashing: $FileName ($FileSize MiB)\n"
#2) The progress of the hash of the file ran through `pv`, with the size precalculated for file progress percents
#3) "Finished $TotalProgressPercent% ($ProcessedBytes/$TotalBytes MiB)\n\n"

#Get $Outfile from the first argument and the $FileList from STDIN (newline delimited)
OutFile="$1";
FileList=`cat /dev/stdin`

#Format a byte count in MegaBytes with comma grouping and 3 decimal places
MbFmtNoExt ()
{
	echo "scale=3; $1/1024/1024" | bc | echo -n `xargs numfmt --grouping`
}

#Add " MiB" to the end of MbFmtNoExt
MbFmt ()
{
	echo `MbFmtNoExt $1`" MiB"
}

#Calculate and output the total size of the file list
echo -n "Calculating total size: "
TotalSize=`echo "$FileList" | xargs -d"\n" stat --printf="%s\n" | paste -s -d+ | bc`
MbFmt $TotalSize
echo #Add an extra newline

#Run over the list (newline delimited)
CalculatedBytes=0
IFS=$'\n'
for FileName in `echo "$FileList"`
do
	#Output the file size and name to STDOUT
	FileSize=`stat --printf="%s" "$FileName"`
	echo "Hashing: $FileName ("`MbFmt $FileSize`")"

	#Output the filename to $OutFile
	echo -n $FileName$'\t' >> $OutFile

	#Run the md5 calculation with `pv` progress
	#Output the hash to $OutFile after the FileName and a tab
	cat "$FileName" | pv -s $FileSize | md5sum | awk '{print $1}' >> $OutFile

	#Output the current progress for the entire file list
	#Format: "Finished $TotalProgressPercent% ($ProcessedBytes/$TotalBytes MiB)\n\n"
	CalculatedBytes=$(($CalculatedBytes+$FileSize))
	echo -n "Finished "
	printf "%.3f" `echo "scale=4; $CalculatedBytes*100/$TotalSize" | bc`
	echo "% ("`MbFmtNoExt $CalculatedBytes`"/"`MbFmt $TotalSize`$')\n'
done

The following is a simple bash script to ping a different domain once a second and log the output. By default, it pings #.castledragmire.com, where # is an incrementing number starting from 0.

The script is written for Cygwin (See the PING_COMMAND variable at the top) but is very easily adaptable to Linux.

The log output is: EPOCH_TIMESTAMP DOMAIN PING_OUTPUT

#This uses Window's native ping since the Cygwin ping is sorely lacking in options
#"-n 1"=Only runs once, "-w 3000"=Timeout after 3 seconds
#The grep strings are also directly tailored for Window's native ping
PING_COMMAND=$(
	echo 'C:/Windows/System32/PING.EXE -n 1 -w 3000 $DOMAIN |';
	echo 'grep -iP "^(Request timed out|Reply from|Ping request could not find)"';
)

i=0 #The subdomain counter
STARTTIME=`date +%s.%N` #This holds the timestamp of the end of the previous loop

#Infinite loop
while true
do
	#Get the domain to run. This requires a domain that has a wildcard as a primary subdomain
	DOMAIN="$i.castledragmire.com"

	#Output the time, domain name, and ping output
	echo `date +%s` "$DOMAIN" $(eval $PING_COMMAND)

	#If less than a second has passed, sleep up to 1 second
	ENDTIME=`date +%s.%N`
	SLEEPTIME=$(echo "1 - ($ENDTIME - $STARTTIME)" | bc)
	STARTTIME=$ENDTIME
	if [ $(echo "$SLEEPTIME>0" | bc) -eq 1 ]; then
		sleep $SLEEPTIME
		STARTTIME=$(echo "$STARTTIME + $SLEEPTIME" | bc)
	fi

	#Increment the subdomain counter
	let i+=1
done

Following is some C++ source code for a Windows kernel-driver service loader. It could be used to load other service types too by changing the dwServiceType flag on the CreateService call. I threw this together for another project I am currently working on. It is also used in the following post (posting soon).

• It is a command line utility which takes 3 arguments:
  1. The service name. Hereby referred to as SERVICE_NAME
  2. The service display name. Hereby referred to as DISPLAY_NAME
  3. The driver path (to the .sys file). Hereby referred to as DRIVER_PATH
• This program (most likely) requires administrative access. There are also some caveats regarding driver code signing requirements that are thoroughly explored elsewhere.
• It first checks to see if a service already exists with the given SERVICE_NAME. If it does:
  1. If the DISPLAY_NAME matches, the service is kept as is.
  2. If the DISPLAY_NAME does not match, the user is prompted on if they want to delete the current service. If they do not, the program exits.
• If the service needs to be created (it did not already exist or was deleted), it creates the service with the given SERVICE_NAME, DISPLAY_NAME, and DRIVER_PATH. If the service is not created during this run, the DRIVER_PATH is ignored.
  Note: The DRIVER_PATH must be to a direct local file system file. I have found that network links and symbolic links do not work.
• The service is started up:
  • If it is already running, the user is prompted on if they want to stop the currently running service. If they say no, the program exits.
• The program then waits for a final user input on if they want to close the service before exiting the program.
• If there was an error, the program reports the error, otherwise, it reports “Success”.
• The program pauses at the end until the user presses any key to exit.
• The program returns 0 on success, and 1 if an error occurred.

//Compiler flags
#define WIN32_LEAN_AND_MEAN  //Include minimum amount of windows stuff
#ifndef _UNICODE //Everything in this script is unicode
	#define _UNICODE
#endif

#include <windows.h>
#include <stdio.h>
#include <conio.h>
#include <memory>

//Smart pointers
typedef std::unique_ptr<WCHAR, void(*)(WCHAR*)> SmartWinAlloc;
typedef std::unique_ptr<SC_HANDLE, void(*)(SC_HANDLE*)> SmartCloseService;
void Delete_SmartWinAlloc(WCHAR *p) { if(p) LocalFree(p); }
void Delete_SmartCloseService(SC_HANDLE *h) { if(h && *h) CloseServiceHandle(*h); }

//Function declarations
WCHAR* InitDriver(int argc, WCHAR *argv[]);
WCHAR* FormatError(WCHAR* Format, ...);
SmartWinAlloc GetLastErrorStr();
BOOLEAN AskQuestion(WCHAR* Question); //Returns if user answered yes

int wmain(int argc, WCHAR *argv[])
{
	//Run the init routine
	WCHAR* Ret=InitDriver(argc, argv);

	//If there is an error, report it, or otherwise, report success
	wprintf(L"%s\n", Ret ? Ret : L"Success");
	wprintf(L"%s\n", L"Press any key to exit");
	_getch();

	//Return if successful
	return (Ret ? 1 : 0);
}

WCHAR* InitDriver(int argc, WCHAR *argv[])
{
	//Confirm arguments
	if(argc<4)
		return FormatError(L"%s", L"3 arguments are required: Service Name, Display Name, Driver Path");
	const WCHAR* Param_ServiceName=argv[1];
	const WCHAR* Param_DisplayName=argv[2];
	const WCHAR* Param_DriverPath =argv[3];

	//Open the service manager
	wprintf(L"%s\n", L"Opening the service manager");
	SC_HANDLE HSCManager=OpenSCManager(nullptr, nullptr, SC_MANAGER_CREATE_SERVICE);
	if(!HSCManager)
		return FormatError(L"%s: %s", L"Error opening service manager", GetLastErrorStr());
	SmartCloseService FreeHSCManager(&HSCManager, Delete_SmartCloseService);

	//Check if the service already exists
	wprintf(L"%s\n", L"Checking previously existing service state");
	BOOL ServiceExists=false;
	{
		//Get the service name
		const DWORD NameBufferSize=255;
		WCHAR NameBuffer[NameBufferSize];
		WCHAR *NamePointer=NameBuffer;
		DWORD NamePointerSize=NameBufferSize;
		std::unique_ptr<WCHAR> Buf(nullptr); //May be swapped with a real pointer later
		for(INT_PTR i=0;i<2;i++)
		{
			//If we found the service, exit the lookup here
			if(GetServiceDisplayName(HSCManager, Param_ServiceName, NamePointer, &NamePointerSize))
			{
				ServiceExists=true;
				break;
			}

			//If the service does not exist, we can exit the lookup here
			if(GetLastError()==ERROR_SERVICE_DOES_NOT_EXIST)
				break;

			//If error is not insufficient buffer size, return the error
			if(GetLastError()!=ERROR_INSUFFICIENT_BUFFER)
				return FormatError(L"%s: %s", L"Could not query service information", GetLastErrorStr());

			//If second pass, error out
			if(i==1)
				return FormatError(L"%s: %s", L"Could not query service information", L"Second buffer pass failed");

			//Create a buffer of appropriate size (and make sure it will later be released)
			NamePointer=new WCHAR[++NamePointerSize];
			std::unique_ptr<WCHAR> Buf2(NamePointer);
			Buf.swap(Buf2);
		}

		//If the service already exists, confirm the service name matches, and if not, ask if user wants to delete the current service
		if(ServiceExists)
		{
			wprintf(L"%s\n", L"The service already exists");
			if(wcsncmp(NamePointer, Param_DisplayName, NamePointerSize+1))
			{
				//If the server names do not match, ask the user what to do
				wprintf(L"%s:\nCurrent: %s\nRequested: %s\n", L"The service names do not match", NamePointer, Param_DisplayName);

				//Make the request
				if(!AskQuestion(L"Would you like to replace the service? (y/n)")) //If user does not wish to replace the service
					return FormatError(L"%s", L"Cannot continue if service names do not match");

				//Delete the service
				wprintf(L"%s\n", L"Deleting the old service");
				ServiceExists=false;
				SC_HANDLE TheService=OpenService(HSCManager, Param_ServiceName, DELETE);
				if(!TheService)
					return FormatError(L"%s: %s", L"Could not open the service to delete it", GetLastErrorStr());
				SmartCloseService CloseTheService(&TheService, Delete_SmartCloseService); //Close the service handle
				if(!DeleteService(TheService))
					return FormatError(L"%s: %s", L"Could not delete the service", GetLastErrorStr());
				wprintf(L"%s\n", L"The service has been deleted");
			}
		}
	}

	//Create the service
	SC_HANDLE TheService;
	if(!ServiceExists)
	{
		//Confirm the driver path exists
		wprintf(L"%s\n", L"Checking the driver file");
		DWORD FileAttrs=GetFileAttributes(Param_DriverPath);
		if(FileAttrs==INVALID_FILE_ATTRIBUTES)
			return FormatError(L"%s: %s", L"Given path is invalid", GetLastErrorStr());
		if(FileAttrs&FILE_ATTRIBUTE_DIRECTORY)
			return FormatError(L"%s: %s", L"Given path is invalid", L"Path is a folder");

		//Create the service
		wprintf(L"%s\n", L"Creating the service");
		TheService=CreateService(
			HSCManager, Param_ServiceName, Param_DisplayName, 
			SERVICE_START|SERVICE_STOP, 
			SERVICE_KERNEL_DRIVER, SERVICE_DEMAND_START, SERVICE_ERROR_IGNORE,
			Param_DriverPath, nullptr, nullptr, nullptr, nullptr, nullptr);
		if(!TheService)
			return FormatError(L"%s: %s", L"Could not create the service", GetLastErrorStr());

	//Open the service if not creating
	} else {
		TheService=OpenService(HSCManager, Param_ServiceName, SERVICE_START|SERVICE_STOP);
		if(!TheService)
			return FormatError(L"%s: %s", L"Could not open the service", GetLastErrorStr());
	}
	SmartCloseService CloseTheService(&TheService, Delete_SmartCloseService); //Close the service on exit

	//Start the service
	wprintf(L"%s\n", L"Starting the service");
	for(INT_PTR i=0;i<2;i++)
	{
		if(StartService(TheService, 0, nullptr))
			break;

		//If not "service already running" error, or user does not want to stop the current service
		if(i==1 || GetLastError()!=ERROR_SERVICE_ALREADY_RUNNING || !AskQuestion(L"The service is already running. Would you like to stop it? (y/n)"))
			return FormatError(L"%s: %s", L"Could not start the service", GetLastErrorStr());

		//Stop the service
		SERVICE_STATUS ss;
		wprintf(L"%s\n", L"Stopping the current service");
		if(!ControlService(TheService, SERVICE_CONTROL_STOP, &ss))
			return FormatError(L"%s: %s", L"Could not stop the current service", GetLastErrorStr());
	}
	wprintf(L"%s\n", L"Started the service");

	//Ask if the user wants to close the service
	if(!AskQuestion(L"Would you like to stop the service before exit? (y/n)"))
		return nullptr;

	//Stop the service
	SERVICE_STATUS ss;
	if(!ControlService(TheService, SERVICE_CONTROL_STOP, &ss))
		return FormatError(L"%s: %s", L"Could not stop the service", GetLastErrorStr());
	if(ss.dwCurrentState!=SERVICE_STOP_PENDING && ss.dwCurrentState!=SERVICE_STOPPED)
		return FormatError(L"%s", L"The service does not appear to be closing");
	wprintf(L"%s\n", L"The service has been stopped");

	//Return success
	return nullptr;
}

WCHAR* FormatError(WCHAR* Format, ...)
{
	static WCHAR Err[255];
	va_list VAList;
	va_start(VAList, Format);
	vswprintf(Err, sizeof(Err)/sizeof(Err[0]), Format, VAList);
	return Err;
}

SmartWinAlloc GetLastErrorStr()
{
	LPWSTR MessageBuffer=nullptr;
	FormatMessage(
		FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS|FORMAT_MESSAGE_MAX_WIDTH_MASK,
		nullptr, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPWSTR)&MessageBuffer, 0, nullptr);
	return SmartWinAlloc(MessageBuffer, Delete_SmartWinAlloc);
}

BOOLEAN AskQuestion(WCHAR* Question)
{
	//Make the request and wait for an input character
	while(1)
	{
		//Ask the question and get the answer
		wprintf(L"%s:", Question);
		fflush(stdout);
		char InputChar=_getch();
		printf("\n");

		//Check for a valid answer
		if(InputChar=='n' || InputChar=='N')
			return FALSE;
		if(InputChar=='y' || InputChar=='Y')
			return TRUE;
	}
}

• Takes raw input and wraps long lines to indent against the current line start.
• When an indented/list line is encountered, which starts with spaces followed by a star "*", wrapped line’s indents will start 2 spaces after the star.
• Lines attempt to split at words of 10 characters or less (see .MinCharsInSplitWord).
• If a line needs to split along a word longer than this, a hyphen is inserted at the end of the line.

import argparse
import re
class BulletHelpFormatter(argparse.HelpFormatter):
    def __init__(self, *args, **kwargs):
        super(BulletHelpFormatter, self).__init__(*args, **kwargs)
        self.MinCharsInSplitWord=10

    def _split_lines(self, text, width):
        #Split lines around line breaks and then modify each line
        Lines=[]
        for Line in text.splitlines():
            #Get the number of spaces to put at subsequent lines
            #0 if not a list item, oherwise, 2+list item start
            ListEl=re.match(r'^ *\*', Line)
            NumBeginningSpace=(0 if ListEl==None else ListEl.end()+1)

            #Add extra spaces at the beginning of each line to match the start of the current line, and go to a maxium of $width
            IsFirstPass=True
            SpacesToAdd=''
            NumSpacesToAdd=0
            while(True):
                #Get the word break points before and after where the line would end
                MaxLineLen=max(min(width-NumSpacesToAdd, len(Line)), 1)
                PrevWordBreak=CurWordBreak=0
                for WordBreak in re.finditer(r'(?<=\W).|\W|$', Line):
                    PrevWordBreak=CurWordBreak
                    CurWordBreak=WordBreak.start()
                    if CurWordBreak>=MaxLineLen:
                        if CurWordBreak==MaxLineLen:
                            PrevWordBreak=CurWordBreak
                        break

                #If previous wordbreak is more than MinCharsInSplitWord away from MaxLineLen, then split at the end of the line
                IsSplit=(PrevWordBreak<1 or CurWordBreak-PrevWordBreak>self.MinCharsInSplitWord)
                SplitPos=(MaxLineLen if IsSplit else PrevWordBreak)

                #Append the new line to the list of lines
                Lines.append(SpacesToAdd+Line[0:SplitPos]+('-' if IsSplit else ''))
                Line=Line[SplitPos:]

                #If this is the end, nothing left to do
                if len(Line)==0:
                    break

                #If this is the first pass, update line creation variables
                if IsFirstPass:
                    IsFirstPass=False
                    NumSpacesToAdd=NumBeginningSpace
                    SpacesToAdd=(' ' * NumSpacesToAdd)

        return Lines

Cygwin has had a long time problem that, depending on your configuration, may cause you to be unable to send a SIGINT (interrupt signal via Ctrl+C) to a native Windows command line executables. As a matter of fact, trying to do so may completely freeze up the console, requiring a process kill of the actual console, bash, and the executable you ran. This problem can crop up for many reasons including the version of Cygwin you are running and your terminal emulator. I specifically installed mintty as my default Cygwin console to get rid of this problem a long time ago (among many other features it had), and now it even has this problem.

While my normal solution is to try and steer clear of native Windows command line executables in Cygwin, this is not always an option. Golang was also causing me this problem every time I ran a network server, which was especially problematic as I would have to ALSO manually kill the server process or it would continue to hold the network port so another test of the code could not use it. An example piece of code is as follows:

package main
import ( "net/http"; "fmt" )
func main() {
	var HR HandleRequest
	if err := http.ListenAndServe("127.0.0.1:81", HR); err!=nil {
		fmt.Println("Error starting server") }
}

//Handle a server request
type HandleRequest struct{}
func (HR HandleRequest) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	fmt.Printf("Received connection from: %s\n", req.RemoteAddr)
}

go run example.go

go build example.go && example.exe

func ListenForExitCommand() {
	for s:=""; s!="exit"; { //Listen for the user to type exit
		if _, err:=fmt.Scanln(&s); err!=nil {
			if err.Error()!="unexpected newline" {
				fmt.Println(err) }
		} else if s=="flush" || s=="exit" {
			//Clean up everything here
		}
	}
	fmt.Println("Exit received, closing process")
	os.Exit(1)
}

go ListenForExitCommand() //Listen for "exit" command

TARGET=`echo $1 | perl -pe 's/\\//\\\\/g'`; #Determine the target from the first parameter (where we are linking to). Change forward slashes to back slashes
LINK=`echo $2 | perl -pe 's/\\//\\\\/g'` #Determine the link name from the second parameter (where the symlink is made). Change forward slashes to back slashes
cmd /c mklink $3 $4 $5 $6 "$LINK" "$TARGET" #Perform the windows mklink command with optional extra parameters

• /d : directory symlink (more hard)
• /j : directory junction (more soft)

• No longer limited to just 4 extra parameters (dynamic instead of static)
• Can now just specify your directory as the link location, and the filename will be automatically filled in
• Handles spaces better

#Get the target and link
TARGET="$1"
shift
LINK="$1"
shift

#If the link is already a directory, append the filename to the end of it
if [ -d "$LINK" ]; then
    #Get the file/directory name without the rest of the path
    ELEMENT_NAME=`echo "$TARGET" | perl -pe 's/^.*?\/([^\/]+)\/?$/$1/'`

    #Append the file name to the target, making sure there is only 1 separating "/"
    LINK=`echo "$LINK" | perl -pe 's/^(.*?)\/?$/$1/'`
    LINK="$LINK"/"$ELEMENT_NAME"
fi

#Replace forward slashes with back slashes
TARGET=`echo $TARGET | perl -pe 's/\\//\\\\/g'`
LINK=`echo $LINK | perl -pe 's/\\//\\\\/g'`

#Perform the windows mklink command with optional extra parameters
cmd /c mklink "$@" "$LINK" "$TARGET"

Rsync is a spectacular bash utility for doing file copying and/or syncing operations. It has a multitude of switches to help optimize and handle any requirements for file copy operation over a local computer or network. However, sometimes networks are less than stable and stalls can happen during an rsync (or scp). This is quite the nuisance when doing very large (i.e. GB) transfers. To solve this, the following script can be used to auto resume a stalled rsync.

export Result=1;
while [ $Result -ne 0 ]; do
  echo "STARTING ($Result) @" `date`;
  rsync -Pza --timeout=10 COPY_FROM COPY_TO_USER@COPY_TO_HOST:COPY_TO_LOCATION;
  Result=$?;
  sleep 1;
done

• The -P switch is highly suggested as it activates:
  • --partial: This keeps a file even if it doesn’t finish transferring so it can be resumed when rsync is restarted. This is especially important if you have very large files.
  • --progress: This shows you the progress of the current file copy operation.
• The -z switch turns on gzip compression during the file transfer, so may only be useful depending on the circumstances.
• The -a switch stands for “archive” and is generally a good idea to use. It includes the switches:
  • -r: Recurse into folders
  • -t: Preserves file modification time stamp. This is highly recommended for this, and incremental backups, as rsync, by default, skips files whose file sizes and modification times match.
  • -l and -D: Preserve file type (i.e. symlinks)
  • -p: Preserve file (chmod) permissions
  • -g and -o: Preserve file owners.
• The --timeout is the crux of the script, in that if an I/O timeout of 10 seconds occurs, the rsync exits prematurely so it can be restarted.

For more useful switches and information, see the rsync man page.

export Result=1; #This will hold the result of the rsync. Set to 1 so the first loop check will fail.
while [ $Result -ne 0 ]; do #Loop until rsync result is successful
  echo "STARTING ($Result) @" `date`; #Inform the user of the time an rsync is starting and the last rsync failure code
  rsync -Pza --timeout=10 COPY_FROM COPY_TO_USER@COPY_TO_HOST:COPY_TO_LOCATION; #See rest of post for switch information
  Result=$?; #Store the result of the rsync
  sleep 1; #This is an optional 1 second timeout between attempts
done

I had the need to pass a program’s [standard] output to a web browser in real time. The best solution for this is to use a combination of programs made in different languages. The following are all of these individual components to accomplish this task.

Please note the C components are only compatible with gcc and bash (cygwin required for Windows), as MSVC and Windows command prompt are missing vital functionality for this to work.

The first component is a server made in C that receives stdin (as a pipe, or typed by the user after line breaks) and sends that data out to a connected client (buffering the output until the client connects).

PassThruServer source, PassThruServer compiled Windows executable.

• This compiles as C99 under gcc:

  gcc PassThruServer.c -o PassThruServer

• Define “WINDOWS” when compiling in Windows (pass “-DWINDOWS”)

#include <stdio.h>
#include <malloc.h>
#include <fcntl.h>
#include <sys/types.h> 
#include <sys/socket.h>
#include <netinet/in.h>
#include <signal.h>

//The server socket and options
int ServerSocket=0;
const int PortNumber=1234; //The port number to listen in on

//If an error occurs, exit cleanly
int error(char *msg)
{
	//Close the socket if it is still open
	if(ServerSocket)
		close(ServerSocket);
	ServerSocket=0;

	//Output the error message, and return the exit status
	fprintf(stderr, "%s\n", msg);
	return 1;
}

//Termination signals
void TerminationSignal(int sig)
{
	error("SIGNAL causing end of process");
	_exit(sig);
}

int main(int argc, char *argv[])
{
	//Listen for termination signals
	signal(SIGINT, TerminationSignal);
	signal(SIGTERM, TerminationSignal);
	signal(SIGHUP, SIG_IGN); //We want the server to continue running if the environment is closed, so SIGHUP is ignored -- This doesn't work in Windows
	
	//Create the server
	struct sockaddr_in ServerAddr={AF_INET, htons(PortNumber), INADDR_ANY, 0}; //Address/port to listen on
	if((ServerSocket=socket(AF_INET, SOCK_STREAM, 0))<0) //Attempt to create the socket
		return error("ERROR on 'socket' call");
	if(bind(ServerSocket, (struct sockaddr*)&ServerAddr, sizeof(ServerAddr))<0) //Bind the socket to the requested address/port
		return error("ERROR on 'bind' call");
	if(listen(ServerSocket,5)<0) //Attempt to listen on the requested address/port
		return error("ERROR on 'listen' call");

	//Accept a connection from a client
	struct sockaddr_in ClientAddr;
	int ClientAddrLen=sizeof(ClientAddr);
	int ClientSocket=accept(ServerSocket, (struct sockaddr*)&ClientAddr, &ClientAddrLen);
	if(ClientSocket<0) 
		return error("ERROR on 'accept' call");

	//Prepare to receive info from STDIN
		//Create the buffer
		const int BufferSize=1024*10;
		char *Buffer=malloc(BufferSize); //Allocate a 10k buffer
		//STDIN only needs to be set to binary mode in windows
		const int STDINno=fileno(stdin);
		#ifdef WINDOWS
			_setmode(STDINno, _O_BINARY);
		#endif
		//Prepare for blocked listening (select function)
		fcntl(STDINno, F_SETFL, fcntl(STDINno, F_GETFL, 0)|O_NONBLOCK); //Set STDIN as blocking
		fd_set WaitForSTDIN;
		FD_ZERO(&WaitForSTDIN);
		FD_SET(STDINno, &WaitForSTDIN);

	//Receive information from STDIN, and pass directly to the client
	int RetVal=0;
	while(1)
	{
		//Get the next block of data from STDIN
		select(STDINno+1, &WaitForSTDIN, NULL, NULL, NULL); //Wait for data
		size_t AmountRead=fread(Buffer, 1, BufferSize, stdin); //Read the data
		if(feof(stdin) || AmountRead==0) //If input is closed, process is complete
			break;
		
		//Send the data to the client
		if(write(ClientSocket,Buffer,AmountRead)<0) //If error in network connection occurred
		{
			RetVal=error("ERROR on 'write' call");
			break;
		}
	}
	
	//Cleanup
	if(ServerSocket)
		close(ServerSocket);
	free(Buffer);
	
	return RetVal;
}

The next component is a Flash applet as the client to receive data. Flash is needed as it can keep a socket open for realtime communication. The applet receives the data and then passes it through to JavaScript for final processing.

Compiled Flash Client Applet

import flash.external.ExternalInterface;
import flash.events.Event;
ExternalInterface.addCallback("OpenSocket", OpenSocket);

function OpenSocket(IP:String, Port:Number):void
{
	SendInfoToJS("Trying to connect");
	var TheSocket:Socket = new Socket();
	TheSocket.addEventListener(Event.CONNECT, function(Success) { SendInfoToJS(Success ? "Connected!" : "Could not connect"); });
	TheSocket.addEventListener(Event.CLOSE, function() { SendInfoToJS("Connection Closed"); });
	TheSocket.addEventListener(IOErrorEvent.IO_ERROR, function() {SendInfoToJS("Could not connect");});
	TheSocket.addEventListener(ProgressEvent.SOCKET_DATA, function(event:ProgressEvent):void { ExternalInterface.call("GetPacket", TheSocket.readUTFBytes(TheSocket.bytesAvailable)); });
	TheSocket.connect(IP, Port);
}
function SendInfoToJS(str:String) { ExternalInterface.call("GetInfoFromFlash", str); }
stop();

var NewSock=new XMLSocket();
NewSock.onData=function(msg) { GetPacket(msg); }
NewSock.onConnect=function(Success) { SendInfoToJS(Success ? "Connected!" : "Could not connect"); }
SendInfoToJS(NewSock.connect(IP, Port) ? "Trying to Connect" : "Could not start connecting");

JavaScript can then receive (and send) information from (and to) the Flash applet through the following functions.

• FLASH.OpenSocket(String IP, Number Port): Call this from JavaScript to open a connection to a server. Note the IP MIGHT have to be the domain the script is running on for security errors to not be thrown.
• JAVASCRIPT.GetInfoFromFlash(String): This is called from Flash whenever connection information is updated. I have it giving arbitrary strings ATM.
• JAVASCRIPT.GetPacket(String): This is called from Flash whenever data is received through the connection.

This example allows the user to input the IP to connect to that is streaming the output. Connection information is shown in the “ConnectionInfo” DOM object. Received data packets are appended to the document in separate DOM objects.

JavaScript+HTML Source

var isIE=navigator.appName.indexOf("Microsoft")!=-1;
function getFlashMovie(movieName) { return (isIE ? window[movieName] : document[movieName]);  }
function $(s) { return document.getElementById(s); }

function Connect()
{
	getFlashMovie("client").OpenSocket($('IP').value, 1234);
}

function GetInfoFromFlash(Str)
{
	$('ConnectionInfo').firstChild.data=Str;
}

function GetPacket(Str)
{
	var NewDiv=document.createElement('DIV');
	NewDiv.appendChild(document.createTextNode(Str));
	$('Info').appendChild(NewDiv);
}

Next is an example application that outputs to stdout. It is important that it flushes stdout after every output or the communication may not be real time.

inc source, inc compiled Windows executable.

inc counts from 0 to one less than a number (parameter #1 [default=50]) after a certain millisecond interval (parameter #2 [default=500]).

./inc 10 #Counts from 0-9 every half a second

#include <stdio.h>
#include <stdlib.h>

int main(int argc, char *argv[])
{
	int NumLoops=(argc>1 ? atoi(argv[1]) : 50); //Number of loops to run from passed argument 1. Default is 50 if not specified.
	int LoopWait=(argc>2 ? atoi(argv[2]) : 500); //Number of milliseconds to wait in between each loop from passed argument 2. Default is 500ms if not specified.
	LoopWait*=1000; //Convert to microseconds for usleep

	//Output an incremented number every half a second
	int i=0;
	while(i<NumLoops)
	{
		printf("%u\n", i++);
		fflush(stdout); //Force stdout flush
		usleep(LoopWait); //Wait for half a second
	};
	
	return 0;
}

This final component is needed so the Flash applet can connect to a server. Unfortunately, new versions of Flash (at least version 10, might have been before that though) started requiring policies for socket connections >:-(. I don’t think this is a problem if you compile your applet to target an older version of Flash with the ActionScript v1.0 code.

This Perl script creates a server on port 843 to respond to Flash policy requests, telling any Flash applet from any domain to allow connections to go through to any port on the computer (IP). It requires Perl, and root privileges on Linux to bind to a port <1024 (su to root or run with sudo).

Flash Socket Policy Server (Rename extension to .pl)

#!/usr/bin/perl
use warnings;
use strict;

#Listen for kill signals
$SIG{'QUIT'}=$SIG{'INT'}=$SIG{__DIE__} = sub
{
	close Server;
	print "Socket Policy Server Ended: $_[0]\n";
	exit;
};

#Start the server:
use Socket;
use IO::Handle;
my $FlashPolicyPort=843;
socket(Server, PF_INET, SOCK_STREAM, getprotobyname('tcp')) or die "'socket' call: $!"; #Open the socket
setsockopt(Server, SOL_SOCKET, SO_REUSEADDR, 1) or die "'setsockopt' call: $!"; #Allow reusing of port/address if in TIME_WAIT state
bind(Server, sockaddr_in($FlashPolicyPort,INADDR_ANY)) or die "'bind' call: $!"; #Listen on port $FlashPolicyPort for connections from any INET adapter
listen(Server,SOMAXCONN) or die "'listen' call: $!"; #Start listening for connections
Server->autoflush(1); #Do not buffer output

#Infinite loop that accepts connections
$/ = "\0"; #Reset terminator from new line to null char
while(my $paddr=accept(Client,Server))
{
	Client->autoflush(1); #Do not buffer IO
	if(<Client> =~ /.*policy\-file.*/i) { #If client requests policy file...
		print Client '<cross-domain-policy><allow-access-from domain="*" to-ports="*" /></cross-domain-policy>'.$/; #Output policy info: Allow any flash applets from any domain to connect
	}
	close Client; #Close the client
}

1. Start the servers
   • In your [bash] command shell, execute the following

     Server/FlashSocketPolicy.pl & #Run the Flash Policy Server as a daemon. Don't forget sudo in Linux
     ./inc | ./PassThruServer #Pipe inc out to the PassThruServer

   • Note that this will immediately start the PassThruServer receiving information from “inc”, so if you don’t get the client up in time, it may already be done counting and send you all the info at once (25 seconds).
   • The PassThruServer will not end until one of the following conditions has been met:
     • The client has connected and the piped process is completed
     • The client has connected and disconnected and the disconnect has been detected (when a packet send failed)
     • It is manually killed through a signal
   • The Flash Policy Server daemon should probably just be left on indefinitely in the background (it only needs to be run once).
2. To run the client, open client.html through a web server [i.e. Apache’s httpd] in your web browser. Don’t open the local file straight through your file system, it needs to be run through a web server for Flash to work correctly.
3. Click “connect” (assuming you are running the PassThruServer already on localhost [the same computer]). You can click “connect” again every time a new PassThruServer is ran.

This is a modification of the utility I made in yesterday’s post, chunk, and most of the information mentioned in that post applies to this one too. This utility instead writes bytes from STDIN to a certain byte offset of a preexisting file. Below is the source code for the result, which I call chunkwrite (Windows Executable).

Chunkwrite writes bytes to a file at a given offset from STDIN. The parameters are:
1) The file to write to
2) The byte offset to write at (hex is supported like 0xA)

//Copyright 2009 by Dakusan (http://www.castledragmire.com/Copyright). Licensed under Dakusan License v2.0 (http://www.castledragmire.com/Misc/Software_Licenses/Dakusan_License_v2.0.php).
//See http://www.castledragmire.com/Posts/chunkwrite for more information

#define __LARGE64_FILES
#include <stdio.h>
#include <stdlib.h> //strtoull

#ifdef WIN32 //STDIN only needs to be set to binary mode in windows
	#include <io.h> //_setmode
	#include <fcntl.h> //_O_BINARY
#endif

typedef unsigned long long UINT64;
const UINT64 MaxSizeToRead=1024*1024*10; //The maximum number of bytes to read at a time to our buffer (Must be < 2^31)

UINT64 GetNumberFromString(const char* S) //Extract both hexidecimal and decimal numbers from a string
{
	bool IsHex=S[0]=='0' && (S[1]|32=='x'); //If string starts as 0x, then is a hex number
	return strtoull(S+(IsHex ? 2 : 0), NULL, IsHex ? 16 : 10); //Hex number starts after 2 characters and uses base 16
}

int main(int argc, char *argv[], char *envp[])
{
	//Determine if proper number of parameters are passed, and if not, output help info
	if(argc!=3)
		return fprintf(stderr, "Chunkwrite writes bytes to a file at a given offset from STDIN. The parameters are:\n1) The file to write to\n2) The byte offset to write at (hex is supported like 0xA)\n") & 0;

	//Open the file to output to
	FILE *TheFile=fopen64(argv[1], "r+b");
	if(TheFile==NULL)
		return fprintf(stderr, "File not found or cannot open file\n") & 0;

	//Determine the requested start offset
	UINT64 Offset=GetNumberFromString(argv[2]);

	//Write the data 10MB at a time from STDIN to the file
	char *Buffer=new char[MaxSizeToRead];
	fseeko64(TheFile, Offset, SEEK_SET); //Seek to the beginning write offset of our file
	#ifdef WIN32 //STDIN only needs to be set to binary mode in windows
		_setmode(_fileno(stdin), _O_BINARY);
	#endif
	do
	{
		size_t AmountRead=fread(Buffer, 1, MaxSizeToRead, stdin); //Read the data from STDIN
		fwrite(Buffer, AmountRead, 1, TheFile); //Write the data to the file
	}
	while(!feof(stdin)); //Keep reading and writing until STDIN is complete

	//Cleanup
	delete[] Buffer;
	fclose(TheFile);
	return 1;
}

I needed a command line utility for Bash (for both Windows and Linux) that only outputs bytes between 2 points in a file to STDOUT. head and tail weren’t really cutting it so I figured I’d throw something together. Below is the source code for the result, which I call chunk (Windows Executable).

I compiled the file as c++, but it should be c99 compatible. The file has been tested as compilable for: GCC4 on Slackware, GCC3 on Red Hat, and GCC3 on MingW (Windows [WIN32 should be defined by the compiler]).

Chunk outputs bytes between 2 points in a file to STDOUT. The parameters are:
1) The file
2) The byte offset to start at (hex is supported like 0xA)
3) The number of bytes to output. If not given, the end of the file is assumed.

//Copyright 2009 by Dakusan (http://www.castledragmire.com/Copyright). Licensed under Dakusan License v2.0 (http://www.castledragmire.com/Misc/Software_Licenses/Dakusan_License_v2.0.php).
//See http://www.castledragmire.com/Posts/chunk for more information

#define __LARGE64_FILES
#include <stdio.h>
#include <stdlib.h> //strtoull

#ifdef WIN32 //STDOUT only needs to be set to binary mode in windows
	#include <io.h> //_setmode
	#include <fcntl.h> //_O_BINARY
#endif

typedef unsigned long long UINT64;
const UINT64 MaxSizeToRead=1024*1024*10; //The maximum number of bytes to read at a time to our buffer (Must be < 2^31)

UINT64 GetNumberFromString(const char* S) //Extract both hexidecimal and decimal numbers from a string
{
	bool IsHex=S[0]=='0' && (S[1]|32=='x'); //If string starts as 0x, then is a hex number
	return strtoull(S+(IsHex ? 2 : 0), NULL, IsHex ? 16 : 10); //Hex number starts after 2 characters and uses base 16
}

int main(int argc, char *argv[], char *envp[])
{
	//Determine if proper number of parameters are passed, and if not, output help info
	if(argc!=3 && argc!=4)
		return fprintf(stderr, "Chunk outputs bytes between 2 points in a file to STDOUT. The parameters are:\n1) The file\n2) The byte offset to start at (hex is supported like 0xA)\n3) The number of bytes to output. If not given, the end of the file is assumed.\n") & 0;

	//Open the file and get its length
	FILE *TheFile=fopen64(argv[1], "rb");
	if(TheFile==NULL)
		return fprintf(stderr, "File not found or cannot open file\n") & 0;
	fseeko64(TheFile, 0, SEEK_END); //Get the length by seeking to the end
	UINT64 FileSize=ftello64(TheFile);

	//Determine the requested start offset
	UINT64 Offset=GetNumberFromString(argv[2]), SizeToOutput;
	if(Offset>=FileSize)
	{
		fprintf(stderr, "Offset is larger than file's size\n");
		fclose(TheFile);
		return 0;
	}

	//Determine the size to read
	if(argc==3) //If no final parameter, read to the end of the file
		SizeToOutput=FileSize-Offset;
	else //Determine from the 3rd parameter
	{
		SizeToOutput=GetNumberFromString(argv[3]);
		if(Offset+SizeToOutput>FileSize)
		{
			fprintf(stderr, "Requested size is larger than the file, truncating to end of file\n");
			SizeToOutput=FileSize-Offset;
		}
		else if(!SizeToOutput) //If nothing to output, exit prematurely
		{
			fclose(TheFile);
			return 1;
		}
	}

	//Output requested data 10MB at a time from the file to STDOUT
	char *Buffer=new char[SizeToOutput>MaxSizeToRead ? MaxSizeToRead : SizeToOutput]; //Only allocate as many bytes to our read buffer as is necessary
	fseeko64(TheFile, Offset, SEEK_SET); //Seek to the beginning read offset of our file
	#ifdef WIN32 //STDOUT only needs to be set to binary mode in windows
		_setmode(_fileno(stdout), _O_BINARY);
	#endif
	while(SizeToOutput) //Keep reading and outputting until requested data is complete
	{
		UINT64 SizeToRead=SizeToOutput>MaxSizeToRead ? MaxSizeToRead : SizeToOutput; //Number of bytes to read and write
		fread(Buffer, SizeToRead, 1, TheFile); //Read the data
		fwrite(Buffer, SizeToRead, 1, stdout); //Write the data to STDOUT
		SizeToOutput-=SizeToRead; //Decrease number of bytes we still need to read
	}

	//Cleanup
	delete[] Buffer;
	fclose(TheFile);
	return 1;
}