tools

lmerge.cpp (9336B)

---

 // Copyright (c) 2020 Morel Bérenger
 // 
 // This software is provided 'as-is', without any express or implied
 // warranty. In no event will the authors be held liable for any damages
 // arising from the use of this software.
 // 
 // Permission is granted to anyone to use this software for any purpose,
 // including commercial applications, and to alter it and redistribute it
 // freely, subject to the following restrictions:
 // 
 // 1. The origin of this software must not be misrepresented; you must not
 //    claim that you wrote the original software. If you use this software
 //    in a product, an acknowledgment in the product documentation would be
 //    appreciated but is not required.
 // 2. Altered source versions must be plainly marked as such, and must not be
 //    misrepresented as being the original software.
 // 3. This notice may not be removed or altered from any source distribution.
 
 #ifdef LIBCPP_MUSL_STATIC
 #define __GLIBC_PREREQ(x,y) 0
 #endif
 
 #include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
 #include <assert.h>
 #include <stdint.h>
 #include <ctype.h>
 
 #include <unistd.h>
 
 #include <algorithm>
 #include <iterator>
 
 /**
  * TODO:
  * * check that ENTRY_SEP works as expected;
  * * fix the fact input needs a "\\n" at end of last line for it to be merged;
  * * UTF-8 support (field separators);
  * * -v/--version option;
  * * allow to customize the memory allocation scheme at runtime;
  * * allow to not print twice merged fields;
  * * allow to set verbosity on stderr;
  * * remove hard-coded limit of UINT16_MAX - 1 for fields start/stop positions;
  * * print as many lines as there where duplicates?
  *
  * Coding rules:
  * * const affect what is before it, so it must follow the type;
  **/
 
 #include <vector.hpp>
 #include <optparser.hpp>
 
 class field_marker;
 typedef vector<char> line_cache;
 typedef vector<field_marker> field_marker_t;
 
 class field_marker
 {
 	uint16_t m_start = UINT16_MAX, m_end = UINT16_MAX;
 
 public:
 	bool ignore( void ) const
 	{
 		return m_start == m_end && m_start == UINT16_MAX;
 	}
 
 	void define( uint16_t start, uint16_t end )
 	{
 		assert( end >= start && start != UINT8_MAX && end != UINT8_MAX );
 		m_start = start; m_end = end;
 	}
 
 	uint16_t start( void ) const
 	{
 		assert( !ignore() );
 		return m_start;
 	}
 
 	uint16_t end( void ) const
 	{
 		assert( !ignore() );
 		return m_end;
 	}
 };
 
 bool allocate_markers(
 		char const * const FIELDS,
 		field_marker_t& field_cache
 );
 
 void print_help( char const* pgm, FILE* target, opt_desc_t const* start, opt_desc_t const* end );
 
 int main( int argc, char **argv )
 {
 	char const * SEP_START = getenv( "FIELD_SEP" );
 	char const * SEP_ENTRY = getenv( "ENTRY_SEP" );
 	char const * FIELDS    = getenv( "FIELDS" );
 	if( !SEP_START )
 	{
 		SEP_START = " \t";
 	}
 	if( !SEP_ENTRY )
 	{
 		SEP_ENTRY = "\n";
 	}
 
 	opt_desc_t opts[] =
 	{
 		STD_HELP,
 #ifndef NO_CMDLINE
 		{ "field_sep", "field separator"  , 't', 0, &SEP_START, set<char const**>, show<char*> },
 		{ "entry_sep", "entry separator"  , 'l', 0, &SEP_ENTRY, set<char const**>, show<char*> },
 		{ "fields"   , "fields to compare", 'f', 0, &FIELDS   , set<char const**>, show<char*> },
 #endif
 	};
 	auto b_opts = std::begin( opts );
 	auto e_opts = std::end( opts );
 
 	char **arg = &argv[1]; assert( argc > 0 );
 	for( int iarg = 1; iarg != argc; ++iarg, ++arg )
 	{
 		auto error = parse_cmd_opt( *arg, b_opts, e_opts );
 		switch( error )
 		{
 			case MAX_COUNT:
 				arg_warning( *arg, error );
 				break;
 			case NONE:
 			case IGNORED:
 				break;
 			case SET_NO_VAL:
 			case SET_VAL_IGN:
 			case SET_FAIL:
 			case BAD_ARGS:
 			case BAD_SETTER:
 				print_help( argv[0], stderr, b_opts, e_opts );
 				arg_error( *arg, error );
 				return EXIT_FAILURE;
 		}
 	}
 
 	if( opts[0].count )
 	{
 		print_help( argv[0], stdout, b_opts, e_opts );
 		return EXIT_SUCCESS;
 	}
 
 	if( !FIELDS )
 	{
 		fputs( "ERROR: FIELDS is not defined\n", stderr );
 		return EXIT_FAILURE;
 	}
 
 	if( strlen( FIELDS ) == 0 )
 	{
 		fputs( "ERROR: FIELDS is empty\n", stderr );
 		return EXIT_FAILURE;
 	}
 
 	field_marker_t field_cache;
 	if( allocate_markers( FIELDS, field_cache ) )
 	{
 		return EXIT_FAILURE;
 	}
 
 	size_t buf_sz = 2048;
 	char* buf = nullptr;
 
 	// allocating a cache of at least 16 bytes.
 	// Note: I don't see how merging lines smaller than 16 bytes can be useful
 	// also, not even enough mem for that would indicate bigger problems...
 	while( !buf && buf_sz >= 32 )
 	{
 		buf_sz /= 2;
 		char* nbuf = static_cast<char*>( realloc( buf, buf_sz ) );
 		if( !nbuf )
 		{
 			free( buf );
 			return EXIT_FAILURE;
 		}
 		buf = nbuf;
 	}
 
 	if( !buf )
 	{
 		fprintf( stderr, "ERROR: malloc %s(%d)\n", strerror( errno ), errno );
 		free( buf );
 		return EXIT_FAILURE;
 	}
 
 	bool fetch = true;
 	line_cache last_line;
 	char const * const SEP_END = SEP_START + strlen( SEP_START );
 	while( !feof( stdin ) )
 	{
 		if( !fgets( buf, static_cast<int>( buf_sz ), stdin ) )
 		{
 			free( buf );
 			buf = nullptr;
 			if( !feof( stdin ) )
 			{
 				fprintf( stderr, "ERROR: fgets %s(%d)\n", strerror( errno ), errno );
 				free( buf );
 				return EXIT_FAILURE;
 			}
 			break;
 		}
 
 		size_t str_sz = strlen( buf );
 		if( str_sz == buf_sz - 1 && buf[str_sz] != '\n' && !feof( stdin ) )
 		{
 			fprintf( stderr, "ERROR: buffer too small for some lines\n" );
 			free( buf );
 			return EXIT_FAILURE;
 		}
 		
 		if( !fetch )
 		{
 			char const* dst_ptr = buf;
 			for( size_t i = 0; i < field_cache.size(); ++i )
 			{
 				field_marker const& src = field_cache[i];
 				if( src.ignore() )
 				{
 					char const *sep = SEP_END;
 					while( sep == SEP_END )
 					{
 						++dst_ptr;
 						sep = SEP_START;
 						for( ; sep != SEP_END && *dst_ptr && *dst_ptr != *sep; ++sep ){}
 					}
 					++dst_ptr;
 					continue;
 				}
 				char const * src_ptr = last_line.data() + src.start();
 				size_t len = src.end() - src.start();
 				if( len > buf_sz - static_cast<size_t>( dst_ptr - buf ) )
 				{
 					fetch = true;
 					break;
 				}
 
 				if( 0 != memcmp( dst_ptr, src_ptr, len ) )
 				{
 					fetch = true;
 					break;
 				}
 
 				char last = dst_ptr[len];
 				char const * sep_ = SEP_START;
 				assert( sep_ != nullptr );
 				while( 0 != *sep_ && *sep_ != last && *SEP_ENTRY != last )
 				{
 					++sep_;
 				}
 				if( 0 == *sep_ )
 				{
 					fetch = true;
 					break;
 				}
 				dst_ptr += len;
 				assert( dst_ptr >= buf );
 			}
 
 			fputc( fetch ? *SEP_ENTRY : * SEP_START, stdout );
 		}
 
 		last_line.assign( buf, buf + str_sz );
 		if( last_line.back() == *SEP_ENTRY )
 		{
 			last_line.back() = 0;
 		}
 
 		if( fetch )
 		{
 			line_cache::iterator start     = last_line.begin();
 			line_cache::iterator cache_end = last_line.end();
 			size_t field_index = 0;
 			while( start != cache_end && field_index < field_cache.size() )
 			{
 				auto end = last_line.end();
 				if( last_line.back() == 0 )
 				{
 					--end;
 				}
 				line_cache::iterator it = std::find_first_of
 					(
 					 start, end,
 					 SEP_START, SEP_END
 					);
 				if( !field_cache[field_index].ignore() )
 				{
 					field_cache[field_index].define(
 							static_cast<uint16_t>( start - last_line.begin() ),
 							static_cast<uint16_t>( it - last_line.begin() )
 							);
 				}
 				start = it + 1;
 				++field_index;
 			}
 			fetch = false;
 		}
 		fputs( last_line.data(), stdout );
 	}
 	fputc( *SEP_ENTRY, stdout );
 	free( buf );
 	return EXIT_SUCCESS;
 }
 
 bool allocate_markers(
 		char const * const FIELDS,
 		field_marker_t& field_cache
 )
 {
 	field_cache.reserve( UINT8_MAX ); //255 fields should fit most cases
 	size_t last_field = 0;
 
 	char const * fields = FIELDS - 1;
 	do
 	{
 		++fields;
 		if( isdigit( *fields ) )
 		{
 			last_field = last_field * 10 + static_cast<size_t>( *fields - '0' );
 		}
 		else if( *fields == ',' || *fields == 0 )
 		{
 			size_t max = std::max( field_cache.size(), last_field );
 			field_cache.resize( max );
 			field_cache[last_field - 1].define( 0, 0 );
 			last_field = 0;
 		}
 		else
 		{
 			fputs( "ERROR: FIELDS contains illegal characters\n", stderr );
 			return true;
 		}
 	}while( *fields );
 	field_cache.shrink_to_fit();
 	return false;
 }
 
 void print_help( char const* pgm, FILE* target, opt_desc_t const* start, opt_desc_t const* end )
 {
 	uint16_t w, h;
 	if( term_ch_size( &w, &h, STDOUT_FILENO ) )
 	{
 		w = 80;
 		fputs( "TODO: could not get terminal's size\n", stderr );
 	}
 	fputs( "Usage: ", target );
 	fputs( pgm, target );
 	fputs( " [OPTIONS]\nDescription:\n", target );
 	char desc[] =
 		"This program reads stdin and when consecutive lines have specific "
 		"fields all containing the same value, prints them replacing the "
 		"newline character by the 1st character in FIELD_SEP.\n"
 		"Fields are delimited by the FIELD_SEP environment variable. If not "
 		"defined, \" \\t\" is used instead (see isblank(3)).\n"
 		"Fields to use are defined by the environment variable FIELDS, which "
 		"only use unsigned decimal integers separated by commas, other "
 		"characters makes the value invalid.\n"
 		"If FIELDS is not defined or invalid, exits with an error.\n"
 		"Empty field indexes (\"1,,3\") are ignored (will resolve in \"1,3\").\n"
 		"Do not work if input is not in line mode.\n"
 		"Line separator is defined by ENTRY_SEP, or \"\\n\" if not defined.\n"
 		;
 	if( indent_txt( desc, std::end( desc ), 1, w, 8, target ) )
 	{
 		fputs( "TODO: handle indent_txt's errors\n", stderr );
 	}
 
 	fputs( "Options:\n", target );
 	print_opts( target, start, end );
 }