From a745159c7d8001491589a912af581ee7ffa2218d Mon Sep 17 00:00:00 2001 From: kx Date: Mon, 23 Oct 2023 21:39:58 +0300 Subject: Bump version to 0.2.2: Update parallel installation process --- README.md | 4 +- configure.ac | 2 +- src/install-package.c | 101 +++++++++++++++++++------------ src/install-pkglist.c | 162 +++++++++++++++++++++++++++++++++++++++++++++++--- src/update-package.c | 78 ++++++++++++++---------- 5 files changed, 265 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 571b711..deb1152 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,10 @@ For example, in this case the **dialog.m4** script will be taken from the On the build machine the installation process seems like that ```Bash -$ tar xJvf pkgtools-0.2.1.tar.xz +$ tar xJvf pkgtools-0.2.2.tar.xz $ mkdir build $ cd build -$ ../pkgtools-0.2.1/configure --prefix=/usr +$ ../pkgtools-0.2.2/configure --prefix=/usr $ make $ make install DESTDIR=$PKG exec_prefix=/ ``` diff --git a/configure.ac b/configure.ac index 89459a0..d76de14 100644 --- a/configure.ac +++ b/configure.ac @@ -7,7 +7,7 @@ AC_PREREQ(2.71) # Minimum Autoconf version required. -AC_INIT([pkgtools], [0.2.1], +AC_INIT([pkgtools], [0.2.2], [support@radix.pro], [pkgtools], [https://radix.pro]) # ============================================================ diff --git a/src/install-package.c b/src/install-package.c index 9b73bf7..cf2a2f9 100644 --- a/src/install-package.c +++ b/src/install-package.c @@ -78,7 +78,7 @@ char *root = NULL, *pkgs_path = NULL, *rempkgs_path = NULL, *pkg_fname = NULL, *asc_fname = NULL, *pkglog_fname = NULL, *pkglist_fname = NULL, *tmpdir = NULL, *curdir = NULL, *log_fname = NULL; -int ask = 0, rqck = 0, gpgck = 0, ignore_chrefs_errors = 0; +int ask = 0, rqck = 0, gpgck = 0, ignore_chrefs_errors = 0, disable_chrefs = 0; char *description = NULL; int exit_status = EXIT_SUCCESS; /* errors counter */ @@ -196,6 +196,7 @@ void usage() fprintf( stdout, " and located in the same directory as the package.\n" ); #endif fprintf( stdout, " --ignore-chrefs-errors Ignore change references errors (code: 48).\n" ); + fprintf( stdout, " --disable-chrefs Do not manage references to the package.\n" ); #if defined( HAVE_DIALOG ) fprintf( stdout, " -i,--info-dialog Show package description during install\n" ); fprintf( stdout, " process using ncurses dialog.\n" ); @@ -683,6 +684,7 @@ void get_args( int argc, char *argv[] ) #endif #define IGNORE_CHREFS_ERRORS 872 +#define DISABLE_CHREFS 873 const struct option long_options[] = { @@ -694,6 +696,7 @@ void get_args( int argc, char *argv[] ) { "gpg-verify", no_argument, NULL, 'g' }, #endif { "ignore-chrefs-errors", no_argument, NULL, IGNORE_CHREFS_ERRORS }, + { "disable-chrefs", no_argument, NULL, DISABLE_CHREFS }, #if defined( HAVE_DIALOG ) { "info-dialog", no_argument, NULL, 'i' }, { "menu-dialog", no_argument, NULL, 'm' }, @@ -758,6 +761,12 @@ void get_args( int argc, char *argv[] ) break; } + case DISABLE_CHREFS: + { + disable_chrefs = 1; + break; + } + case 'p': { if( optarg != NULL ) @@ -1509,8 +1518,8 @@ static void read_service_files( void ) static void check_package( void ) { - pid_t p = (pid_t) -1; - int rc; + pid_t p = (pid_t) -1; + int rc = EXIT_SUCCESS; int len = 0; char *cmd = NULL; @@ -2184,12 +2193,25 @@ static void pre_install_routine( void ) } } +static int __nstreams( void ) +{ + int ret = 1; + int nprocs = get_nprocs(); + + if( nprocs > 4 ) + { + ret = nprocs / 2; + } + + return ret; +} + static const char *fill_decompressor( char *buffer, char compressor ) { switch( compressor ) { case 'J': - (void)sprintf( buffer, "xz --threads=%d -dc", get_nprocs() ); + (void)sprintf( buffer, "xz --threads=%d -dc", __nstreams() ); break; case 'j': (void)sprintf( buffer, "bzip2 -dc" ); @@ -2206,8 +2228,8 @@ static const char *fill_decompressor( char *buffer, char compressor ) static void uncompress_package( void ) { - pid_t p = (pid_t) -1; - int rc; + pid_t p = (pid_t) -1; + int rc = EXIT_SUCCESS; int len = 0; char *cmd = NULL; @@ -2442,48 +2464,51 @@ static void finalize_installation( void ) /********************************************* Increment references in the Setup Database: */ - if( group ) - len = snprintf( &cmd[0], PATH_MAX, - "%s/chrefs --operation=inc --destination=%s %s/%s > /dev/null 2>&1", - selfdir, pkgs_path, group, basename( (char *)pkglog_fname ) ); - else - len = snprintf( &cmd[0], PATH_MAX, - "%s/chrefs --operation=inc --destination=%s %s > /dev/null 2>&1", - selfdir, pkgs_path, basename( (char *)pkglog_fname ) ); - if( len == 0 || len == PATH_MAX - 1 ) + if( !disable_chrefs ) { - FATAL_ERROR( "Cannot increment '%s-%s' package references", pkgname, pkgver ); - } - p = sys_exec_command( cmd ); - rc = sys_wait_command( p, (char *)NULL, PATH_MAX ); + if( group ) + len = snprintf( &cmd[0], PATH_MAX, + "%s/chrefs --operation=inc --destination=%s %s/%s > /dev/null 2>&1", + selfdir, pkgs_path, group, basename( (char *)pkglog_fname ) ); + else + len = snprintf( &cmd[0], PATH_MAX, + "%s/chrefs --operation=inc --destination=%s %s > /dev/null 2>&1", + selfdir, pkgs_path, basename( (char *)pkglog_fname ) ); + if( len == 0 || len == PATH_MAX - 1 ) + { + FATAL_ERROR( "Cannot increment '%s-%s' package references", pkgname, pkgver ); + } + p = sys_exec_command( cmd ); + rc = sys_wait_command( p, (char *)NULL, PATH_MAX ); - free( cmd ); + free( cmd ); - if( (rc != 0) && !ignore_chrefs_errors ) - { - free( tmp ); + if( (rc != 0) && !ignore_chrefs_errors ) + { + free( tmp ); - rollback(); + rollback(); - exit_status = 48; + exit_status = 48; - if( install_mode != CONSOLE ) - { + if( install_mode != CONSOLE ) + { #if defined( HAVE_DIALOG ) - info_pkg_box( "Install:", pkgname, pkgver, strprio( priority, 0 ), - "\n\\Z1Cannot increment package references in Setup Database.\\Zn\n", 5, 0, 0 ); + info_pkg_box( "Install:", pkgname, pkgver, strprio( priority, 0 ), + "\n\\Z1Cannot increment package references in Setup Database.\\Zn\n", 5, 0, 0 ); #else - fprintf( stdout, "\nCannot increment '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); + fprintf( stdout, "\nCannot increment '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); #endif - } - else - { - fprintf( stdout, "\nCannot increment '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); - } + } + else + { + fprintf( stdout, "\nCannot increment '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); + } - if( tmpdir ) { _rm_tmpdir( (const char *)tmpdir ); free( tmpdir ); } - free_resources(); - exit( exit_status ); + if( tmpdir ) { _rm_tmpdir( (const char *)tmpdir ); free( tmpdir ); } + free_resources(); + exit( exit_status ); + } } /************************************************* diff --git a/src/install-pkglist.c b/src/install-pkglist.c index 7e0cc00..26e00e9 100644 --- a/src/install-pkglist.c +++ b/src/install-pkglist.c @@ -70,7 +70,8 @@ #include -#define WAIT_USEC_FOR_CHILD 10000 +#define START_USEC_FOR_CHILD 30000 +#define WAIT_USEC_FOR_CHILD 10000 char *program = PROGRAM_NAME; char *root = NULL, *srcdir = NULL, *pkglist_fname = NULL, @@ -933,6 +934,31 @@ void get_args( int argc, char *argv[] ) FATAL_ERROR( "Defined --root '%s' is not a directory", buf ); } + len = strlen( (const char *)buf ); + + (void)strcat( buf, PACKAGES_PATH ); + if( _mkdir_p( buf, S_IRWXU | S_IRWXG | S_IRWXO ) != 0 ) + { + FATAL_ERROR( "Cannot access '/%s' directory", PACKAGES_PATH ); + } + + /********************************************* + Create other directories of Setup Database: + */ + buf[len] = '\0'; + (void)strcat( buf, REMOVED_PKGS_PATH ); + if( _mkdir_p( buf, S_IRWXU | S_IRWXG | S_IRWXO ) != 0 ) + { + FATAL_ERROR( "Cannot access '/%s' directory", REMOVED_PKGS_PATH ); + } + + buf[len] = '\0'; + (void)strcat( buf, SETUP_PATH ); + if( _mkdir_p( buf, S_IRWXU | S_IRWXG | S_IRWXO ) != 0 ) + { + FATAL_ERROR( "Cannot access '/%s' directory", SETUP_PATH ); + } + free( buf ); } /* @@ -1652,7 +1678,7 @@ static void install_package( struct package *package ) if( gpgck ) (void)sprintf( opt, "--gpg-verify " ); if( (install_mode != CONSOLE) && !parallel && !progress ) (void)strcat( opt, "--info-dialog " ); - if( parallel ) (void)strcat( opt, "--ignore-chrefs-errors " ); + if( parallel ) (void)strcat( opt, "--disable-chrefs " ); if( (install_mode == CONSOLE) && !parallel && !progress ) out = " "; cmd = (char *)malloc( (size_t)PATH_MAX ); @@ -1728,27 +1754,112 @@ static void serial_install_packages( void ) } } +static int __install_count( void ) +{ + struct dlist *list = packages, *next = NULL; + int ret = 0; + + while( list ) + { + struct package *package = NULL; + + next = dlist_next( list ); + package = (struct package *)list->data; + if( package && !strncmp( "install", strproc( package->procedure ), 7 ) ) + { + ++ret; + } + list = next; + } + + return ret; +} + +static int __update_count( void ) +{ + struct dlist *list = packages, *next = NULL; + int ret = 0; + + while( list ) + { + struct package *package = NULL; + + next = dlist_next( list ); + package = (struct package *)list->data; + if( package && !strncmp( "update", strproc( package->procedure ), 6 ) ) + { + ++ret; + } + list = next; + } + + return ret; +} + +static int __nstreams( void ) +{ + int ret = 1; + int nprocs = get_nprocs(); + + if( nprocs > 4 ) + { + ret = nprocs / 2; + } + + return ret; +} static void *install_process( void *args ) { struct dlist *list = packages, *next = NULL; - int nstreams = ncpus * 2; /* two concurents for CPU */ + int nstreams = __nstreams(); + + /********************************************* + install packages with procedure == install: + */ + while( list ) + { + struct package *package = NULL; + + next = dlist_next( list ); + package = (struct package *)list->data; + if( package && !strncmp( "install", strproc( package->procedure ), 7 ) ) + { + install_package( package ); + } + list = next; + + /* wait for available CPU: */ + while( (__child - __terminated) > nstreams ) usleep( START_USEC_FOR_CHILD ); + } + + return NULL; +} + +static void *update_process( void *args ) +{ + struct dlist *list = packages, *next = NULL; + + int nstreams = __nstreams(); + /********************************************* + install packages with procedure == update: + */ while( list ) { struct package *package = NULL; next = dlist_next( list ); package = (struct package *)list->data; - if( package ) + if( package && !strncmp( "update", strproc( package->procedure ), 6 ) ) { install_package( package ); } list = next; /* wait for available CPU: */ - while( (__child - __terminated) > nstreams ) usleep( WAIT_USEC_FOR_CHILD ); + while( (__child - __terminated) > nstreams ) usleep( START_USEC_FOR_CHILD ); } return NULL; @@ -1768,6 +1879,20 @@ static void parallel_install_packages( void ) (void)pthread_detach( install_process_id ); } +static void parallel_update_packages( void ) +{ + pthread_t install_process_id; + int status; + + /* Start the parallel installation process: */ + status = pthread_create( &install_process_id, NULL, update_process, NULL ); + if( status != 0 ) + { + FATAL_ERROR( "Cannot start parallel installation process" ); + } + (void)pthread_detach( install_process_id ); +} + /* End of install functions. *********************************************/ @@ -1939,6 +2064,8 @@ int main( int argc, char *argv[] ) parallel installation: */ int percent = 0; + int __all_install = __install_count(); + int __all_update = __update_count(); __all = dlist_length( packages ); __done = 0; __child = 0; @@ -1947,10 +2074,28 @@ int main( int argc, char *argv[] ) show_progress(); + /********************************************* + install packages with procedure == install: + */ parallel_install_packages(); - if( __terminated < __all ) + while( !__done ) { + percent = ( __terminated < __all ) ? __terminated * 100 / __all : 100; + + update_progress( percent ); + usleep( WAIT_USEC_FOR_CHILD ); + } + + __done = 0; __child = 0; + + if( __all_update ) + { + /********************************************* + install packages with procedure == update: + */ + parallel_update_packages(); + while( !__done ) { percent = ( __terminated < __all ) ? __terminated * 100 / __all : 100; @@ -1958,9 +2103,9 @@ int main( int argc, char *argv[] ) update_progress( percent ); usleep( WAIT_USEC_FOR_CHILD ); } - } - __done = 0; __child = 0; + __done = 0; __child = 0; + } stop_progress(); @@ -2025,7 +2170,6 @@ int main( int argc, char *argv[] ) } - if( tmpdir ) { _rm_tmpdir( (const char *)tmpdir ); free( tmpdir ); } free_resources(); diff --git a/src/update-package.c b/src/update-package.c index d11a97c..b5ce23f 100644 --- a/src/update-package.c +++ b/src/update-package.c @@ -78,7 +78,7 @@ char *root = NULL, *pkgs_path = NULL, *rempkgs_path = NULL, *remlog_fname = NULL *pkg_fname = NULL, *asc_fname = NULL, *pkglog_fname = NULL, *pkglist_fname = NULL, *tmpdir = NULL, *rtmpdir = NULL, *curdir = NULL, *log_fname = NULL; -int ask = 0, rqck = 0, gpgck = 0, reinstall = 0, ignore_chrefs_errors = 0; +int ask = 0, rqck = 0, gpgck = 0, reinstall = 0, ignore_chrefs_errors = 0, disable_chrefs = 0; char *description = NULL; int exit_status = EXIT_SUCCESS; /* errors counter */ @@ -211,6 +211,7 @@ void usage() fprintf( stdout, " and located in the same directory as the package.\n" ); #endif fprintf( stdout, " --ignore-chrefs-errors Ignore change references errors (code: 48).\n" ); + fprintf( stdout, " --disable-chrefs Do not manage references to the package.\n" ); #if defined( HAVE_DIALOG ) fprintf( stdout, " -i,--info-dialog Show package description during update\n" ); fprintf( stdout, " process using ncurses dialog.\n" ); @@ -700,6 +701,7 @@ void get_args( int argc, char *argv[] ) #define REINSTALL 812 #define IGNORE_CHREFS_ERRORS 872 +#define DISABLE_CHREFS 873 const struct option long_options[] = { @@ -711,6 +713,7 @@ void get_args( int argc, char *argv[] ) { "gpg-verify", no_argument, NULL, 'g' }, #endif { "ignore-chrefs-errors", no_argument, NULL, IGNORE_CHREFS_ERRORS }, + { "disable-chrefs", no_argument, NULL, DISABLE_CHREFS }, #if defined( HAVE_DIALOG ) { "info-dialog", no_argument, NULL, 'i' }, { "menu-dialog", no_argument, NULL, 'm' }, @@ -780,6 +783,11 @@ void get_args( int argc, char *argv[] ) ignore_chrefs_errors = 1; break; } + case DISABLE_CHREFS: + { + disable_chrefs = 1; + break; + } case 'p': { @@ -2446,44 +2454,47 @@ static void finalize_removal( void ) /********************************************* Decrement references in the Setup Database: */ - if( installed_group ) - len = snprintf( &cmd[0], PATH_MAX, - "%s/chrefs --operation=dec --destination=%s %s/%s > /dev/null 2>&1", - selfdir, pkgs_path, installed_group, basename( (char *)remlog_fname ) ); - else - len = snprintf( &cmd[0], PATH_MAX, - "%s/chrefs --operation=dec --destination=%s %s > /dev/null 2>&1", - selfdir, pkgs_path, basename( (char *)remlog_fname ) ); - if( len == 0 || len == PATH_MAX - 1 ) + if( !disable_chrefs ) { - FATAL_ERROR( "Cannot decrement '%s-%s' package references", pkgname, pkgver ); - } - p = sys_exec_command( cmd ); - rc = sys_wait_command( p, (char *)NULL, PATH_MAX ); - if( (rc != 0) && !ignore_chrefs_errors ) - { - free( cmd ); - free( tmp ); + if( installed_group ) + len = snprintf( &cmd[0], PATH_MAX, + "%s/chrefs --operation=dec --destination=%s %s/%s > /dev/null 2>&1", + selfdir, pkgs_path, installed_group, basename( (char *)remlog_fname ) ); + else + len = snprintf( &cmd[0], PATH_MAX, + "%s/chrefs --operation=dec --destination=%s %s > /dev/null 2>&1", + selfdir, pkgs_path, basename( (char *)remlog_fname ) ); + if( len == 0 || len == PATH_MAX - 1 ) + { + FATAL_ERROR( "Cannot decrement '%s-%s' package references", pkgname, pkgver ); + } + p = sys_exec_command( cmd ); + rc = sys_wait_command( p, (char *)NULL, PATH_MAX ); + if( (rc != 0) && !ignore_chrefs_errors ) + { + free( cmd ); + free( tmp ); - exit_status = 48; + exit_status = 48; - if( update_mode != CONSOLE ) - { + if( update_mode != CONSOLE ) + { #if defined( HAVE_DIALOG ) - info_pkg_box( "Update:", pkgname, pkgver, NULL, - "\n\\Z1Cannot decrement package references in Setup Database.\\Zn\n", 5, 0, 0 ); + info_pkg_box( "Update:", pkgname, pkgver, NULL, + "\n\\Z1Cannot decrement package references in Setup Database.\\Zn\n", 5, 0, 0 ); #else - fprintf( stdout, "\nCannot decrement '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); + fprintf( stdout, "\nCannot decrement '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); #endif - } - else - { - fprintf( stdout, "\nCannot decrement '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); - } + } + else + { + fprintf( stdout, "\nCannot decrement '%s-%s' package references in Setup Database.\n\n", pkgname, pkgver ); + } - if( tmpdir ) { _rm_tmpdir( (const char *)tmpdir ); free( tmpdir ); } - free_resources(); - exit( exit_status ); + if( tmpdir ) { _rm_tmpdir( (const char *)tmpdir ); free( tmpdir ); } + free_resources(); + exit( exit_status ); + } } /***************************************************** @@ -3074,6 +3085,8 @@ static void finalize_update( void ) /********************************************* Increment references in the Setup Database: */ +if( !disable_chrefs ) +{ if( group ) len = snprintf( &cmd[0], PATH_MAX, "%s/chrefs --operation=inc --destination=%s %s/%s > /dev/null 2>&1", @@ -3117,6 +3130,7 @@ static void finalize_update( void ) free_resources(); exit( exit_status ); } +} /************************************************* Remove backup PKGLOG file from removed-packages -- cgit v1.2.3