[interchange-cvs] interchange - heins modified 5 files

interchange-core@interchange.redhat.com interchange-core@interchange.redhat.com
Sat Dec 29 14:50:00 2001


User:      heins
Date:      2001-12-29 19:49:33 GMT
Modified:  lib/Vend Data.pm DbSearch.pm Glimpse.pm TextSearch.pm
Modified:  lib/Vend/Table Common.pm
Log:
* Add new MIRROR parameter for databases. This allows creation of a
  DBM or Memory mirror.

  Sequence is:

	1. Table is always imported at config time.
	2. Table is only imported once.
	3. Text file is not required, but should be set so export
	   can work properly and so there is a placeholder.

  A complete configuration is:

	  Database  products_mem  products_mem.txt  TAB
	  Database  products_mem  MIRROR  products
	  Database  products_mem  MEMORY  1

  MIRROR name is not set in stone yet, probably not apropos
  enough; suggestions are welcome.

  Benchmarks for 1000 separate (i.e. tag_data, no caching)
  accesses (repeated tries show these are representative):

	Doing products_mem:
	Class=MEMORY DSN=
	Data returned 247 bytes
	TIME=0.200

	Doing products:
	Class=DBI DSN=dbi:mysql:test_mfound
	Data returned 247 bytes
	TIME=0.670

	Doing alias_mem:
	Class=MEMORY DSN=
	Data returned 9 bytes
	TIME=0.180

	Doing alias:
	Class=DBI DSN=dbi:Pg:dbname=wikicms
	Data returned 9 bytes
	TIME=0.700

	Doing wiki_mem:
	Class=GDBM DSN=
	Data returned 400 bytes
	TIME=0.300

	Doing wiki_mem2:
	Class=MEMORY DSN=
	Data returned 400 bytes
	TIME=0.180

	Doing wiki:
	Class=DBI DSN=dbi:Pg:dbname=wikicms
	Data returned 400 bytes
	TIME=0.710

 Gains should be even bigger as no database connections are needed
 on a busy system.

Revision  Changes    Path
2.5       +49 -3     interchange/lib/Vend/Data.pm


rev 2.5, prev_rev 2.4
Index: Data.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/Data.pm,v
retrieving revision 2.4
retrieving revision 2.5
diff -u -r2.4 -r2.5
--- Data.pm	2001/12/28 17:55:44	2.4
+++ Data.pm	2001/12/29 19:49:33	2.5
@@ -1,6 +1,6 @@
 # Vend::Data - Interchange databases
 #
-# $Id: Data.pm,v 2.4 2001/12/28 17:55:44 mheins Exp $
+# $Id: Data.pm,v 2.5 2001/12/29 19:49:33 mheins Exp $
 # 
 # Copyright (C) 1996-2001 Red Hat, Inc. <interchange@redhat.com>
 #
@@ -647,7 +647,22 @@
 	if($Global::Database) {
 		copyref($Global::Database, $Vend::Cfg->{Database});
 	}
-    while (($name,$data) = each %{$Vend::Cfg->{Database}}) {
+
+	my @tables = keys %{$Vend::Cfg->{Database}};
+
+	my @delayed;
+	my $redone;
+
+	TIEDB: {
+
+		foreach $name (@tables) {
+			$data = $Vend::Cfg->{Database}{$name} || {};
+			if(! $redone and $data->{MIRROR}) {
+#::logDebug("mirror database $name, delaying");
+				$data->{HOT} = 1;
+				push @delayed, $name;
+				next;
+			}
 		if(! $data->{name}) {
 #::logDebug("Screwed up database: " . ::uneval( $data) );
 			next;
@@ -679,6 +694,16 @@
 			$Vend::Database{$name} = new $class ($data);
 		}
 	}
+
+		# So mirrors will not happen until after mirror source
+		if(@delayed) {
+			@tables = @delayed;
+			@delayed = ();
+			$redone = 1;
+			redo TIEDB;
+		}
+
+	}
 	update_productbase();
 }
 
@@ -816,6 +841,17 @@
 		}
 	}
 
+	if($obj->{MIRROR}) {
+		if($obj->{Mirror_complete}) {
+			$no_import = 1;
+		}
+		else {
+#::logDebug ("table $new_table_name: undeffing $database_dbm, hot=$obj->{HOT}");
+			undef $database_dbm;
+			undef $no_import;
+		}
+	}
+
 	last IMPORT if $no_import;
 #::logDebug ("moving to import") if $type == 9;
 
@@ -826,6 +862,7 @@
     if (
 		! defined $database_dbm
 		or ! -e $database_dbm
+		or $obj->{MIRROR}
         or ($txt_time = file_modification_time($database_txt, $obj->{PRELOAD}))
 				>
            ($dbm_time = file_modification_time($database_dbm))
@@ -848,11 +885,20 @@
 
 		local($/) = $record_delim if defined $record_delim;
 
+		if($obj->{MIRROR}) {
+			$db = Vend::Table::Common::import_from_ic_db(
+							$database_txt,
+							$obj,
+							$new_table_name,
+				);
+		}
+		else {
         $db = Vend::Table::Common::import_ascii_delimited(
 							$database_txt,
 							$obj,
 							$new_table_name,
 				);
+		}
 
 		$/ = $save;
 		if(defined $database_dbm) {
@@ -1021,7 +1067,7 @@
 	$scan =~ s:^scan/::;
 
 	my $c = {
-				mv_list_only		=> 1,
+				mv_list_only        => 1,
 				mv_search_file		=> $bx_fn,
 			};
 



2.6       +8 -8      interchange/lib/Vend/DbSearch.pm


rev 2.6, prev_rev 2.5
Index: DbSearch.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/DbSearch.pm,v
retrieving revision 2.5
retrieving revision 2.6
diff -u -r2.5 -r2.6
--- DbSearch.pm	2001/11/26 18:34:02	2.5
+++ DbSearch.pm	2001/12/29 19:49:33	2.6
@@ -1,6 +1,6 @@
 # Vend::DbSearch - Search indexes with Interchange
 #
-# $Id: DbSearch.pm,v 2.5 2001/11/26 18:34:02 mheins Exp $
+# $Id: DbSearch.pm,v 2.6 2001/12/29 19:49:33 mheins Exp $
 #
 # Adapted for use with Interchange from Search::TextSearch
 #
@@ -26,7 +26,7 @@
 
 @ISA = qw(Vend::Search);
 
-$VERSION = substr(q$Revision: 2.5 $, 10);
+$VERSION = substr(q$Revision: 2.6 $, 10);
 
 use Search::Dict;
 use strict;
@@ -34,24 +34,24 @@
 sub array {
 	my ($s, $opt) = @_;
 	$s->{mv_one_sql_table} = 1;
-	$s->{mv_list_only} = 1;
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 sub hash {
 	my ($s, $opt) = @_;
 	$s->{mv_return_reference} = 'HASH';
 	$s->{mv_one_sql_table} = 1;
-	$s->{mv_list_only} = 1;
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 sub list {
 	my ($s, $opt) = @_;
-	$s->{mv_list_only} = 1;
 	$s->{mv_return_reference} = 'LIST';
 	$s->{mv_one_sql_table} = 1;
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 my %Default = (



2.3       +8 -8      interchange/lib/Vend/Glimpse.pm


rev 2.3, prev_rev 2.2
Index: Glimpse.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/Glimpse.pm,v
retrieving revision 2.2
retrieving revision 2.3
diff -u -r2.2 -r2.3
--- Glimpse.pm	2001/11/26 18:34:02	2.2
+++ Glimpse.pm	2001/12/29 19:49:33	2.3
@@ -1,6 +1,6 @@
 # Vend::Glimpse - Search indexes with Glimpse
 #
-# $Id: Glimpse.pm,v 2.2 2001/11/26 18:34:02 mheins Exp $
+# $Id: Glimpse.pm,v 2.3 2001/12/29 19:49:33 mheins Exp $
 #
 # Adapted for use with Interchange from Search::Glimpse
 #
@@ -25,27 +25,27 @@
 require Vend::Search;
 @ISA = qw(Vend::Search);
 
-$VERSION = substr(q$Revision: 2.2 $, 10);
+$VERSION = substr(q$Revision: 2.3 $, 10);
 use strict;
 
 sub array {
 	my ($s, $opt) = @_;
-	$s->{mv_list_only} = 1;
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 sub hash {
 	my ($s, $opt) = @_;
 	$s->{mv_return_reference} = 'HASH';
-	$s->{mv_list_only} = 1;
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 sub list {
 	my ($s, $opt) = @_;
-	$s->{mv_list_only} = 1;
 	$s->{mv_return_reference} = 'LIST';
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 my %Default = (



2.3       +8 -8      interchange/lib/Vend/TextSearch.pm


rev 2.3, prev_rev 2.2
Index: TextSearch.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/TextSearch.pm,v
retrieving revision 2.2
retrieving revision 2.3
diff -u -r2.2 -r2.3
--- TextSearch.pm	2001/12/07 04:16:06	2.2
+++ TextSearch.pm	2001/12/29 19:49:33	2.3
@@ -1,6 +1,6 @@
 # Vend::TextSearch - Search indexes with Perl
 #
-# $Id: TextSearch.pm,v 2.2 2001/12/07 04:16:06 jon Exp $
+# $Id: TextSearch.pm,v 2.3 2001/12/29 19:49:33 mheins Exp $
 #
 # Adapted for use with Interchange from Search::TextSearch
 #
@@ -28,29 +28,29 @@
 use vars qw(@ISA);
 @ISA = qw(Vend::Search);
 
-$VERSION = substr(q$Revision: 2.2 $, 10);
+$VERSION = substr(q$Revision: 2.3 $, 10);
 
 use Search::Dict;
 use strict;
 
 sub array {
 	my ($s, $opt) = @_;
-	$s->{mv_list_only} = 1;
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 sub hash {
 	my ($s, $opt) = @_;
 	$s->{mv_return_reference} = 'HASH';
-	$s->{mv_list_only} = 1;
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 sub list {
 	my ($s, $opt) = @_;
-	$s->{mv_list_only} = 1;
 	$s->{mv_return_reference} = 'LIST';
-	Vend::Scan::perform_search($opt, undef, $s);
+	$s->{mv_list_only} = 1; # makes perform_search only return results array
+	return Vend::Scan::perform_search($opt, undef, $s);
 }
 
 my %Default = (



2.8       +62 -2     interchange/lib/Vend/Table/Common.pm


rev 2.8, prev_rev 2.7
Index: Common.pm
===================================================================
RCS file: /anon_cvs/repository/interchange/lib/Vend/Table/Common.pm,v
retrieving revision 2.7
retrieving revision 2.8
diff -u -r2.7 -r2.8
--- Common.pm	2001/12/07 00:20:28	2.7
+++ Common.pm	2001/12/29 19:49:33	2.8
@@ -1,6 +1,6 @@
 # Vend::Table::Common - Common access methods for Interchange databases
 #
-# $Id: Common.pm,v 2.7 2001/12/07 00:20:28 jon Exp $
+# $Id: Common.pm,v 2.8 2001/12/29 19:49:33 mheins Exp $
 #
 # Copyright (C) 1996-2001 Red Hat, Inc. <interchange@redhat.com>
 #
@@ -22,7 +22,7 @@
 # Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 # MA  02111-1307  USA.
 
-$VERSION = substr(q$Revision: 2.7 $, 10);
+$VERSION = substr(q$Revision: 2.8 $, 10);
 use strict;
 
 package Vend::Table::Common;
@@ -1277,6 +1277,66 @@
 		);
 	}
     return $out;
+}
+
+sub import_from_ic_db {
+    my ($infile, $options, $table_name) = @_;
+
+	my $tname = $options->{MIRROR}
+		or die errmsg(
+				"Memory mirror table not specified for table %s.",
+				$table_name,
+			);
+#::logDebug("Importing mirrored $table_name from $tname");
+
+	$Vend::Database{$tname} =
+		Vend::Data::import_database($Vend::Cfg->{Database}{$tname})
+			unless $Vend::Database{$tname};
+
+	my $idb = Vend::Data::database_exists_ref($tname)
+		or die errmsg(
+				"Memory mirror table %s does not exist (yet) to create mirror %s.\n",
+				$tname,
+				$table_name,
+			);
+
+	my @field_names = $idb->columns;
+
+	my $odb;
+
+	if($options->{ObjectType}) {
+		no strict 'refs';
+		$odb = &{"$options->{ObjectType}::create"}(
+									$options->{ObjectType},
+									$options,
+									\@field_names,
+									$table_name,
+								);
+	}
+	else {
+		$odb = $options->{Object};
+	}
+
+#::logDebug("idb=$idb odb=$odb");
+	eval {
+		my $f;
+		while($f = $idb->each_nokey()) {
+#::logDebug("importing key=$f->[0]");
+			$odb->set_row(@$f);
+		}
+	};
+
+	if($@) {
+		die ::errmsg(
+				"Problem with mirror import from source %s to target %s\n",
+				$tname,
+				$table_name,
+				);
+	}
+	
+	$odb->[$CONFIG]{Mirror_complete} = 1;
+	delete $odb->[$CONFIG]{Clean_start};
+    return $odb;
 }
 
 my $white = ' \t';