[interchange-cvs] interchange - heins modified 3 files
interchange-core@icdevgroup.org
interchange-core@icdevgroup.org
Sun Jan 12 13:29:01 2003
User: heins
Date: 2003-01-12 18:28:45 GMT
Modified: code/UI_Tag import_fields.coretag
Modified: lib/Vend/Table DBI.pm Common.pm
Log:
* Add transaction boundary support to import routines, should cut down
Postgres import times dramatically.
* Minor change to quote() routine using Jon's PREFER_NULL.
Revision Changes Path
1.6 +95 -5 interchange/code/UI_Tag/import_fields.coretag
rev 1.6, prev_rev 1.5
Index: import_fields.coretag
===================================================================
RCS file: /var/cvs/interchange/code/UI_Tag/import_fields.coretag,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- import_fields.coretag 1 Nov 2002 19:38:43 -0000 1.5
+++ import_fields.coretag 12 Jan 2003 18:28:45 -0000 1.6
@@ -1,5 +1,6 @@
UserTag import_fields Order table
UserTag import_fields addAttr
+UserTag import_fields Version $Revision: 1.6 $
UserTag import_fields Routine <<EOR
sub {
my($table, $opt) = @_;
@@ -26,6 +27,34 @@
my $tmsg = '';
my $db;
+ my %filter = (
+ '' => { mv_credit_card_number => 'encrypt' },
+ );
+
+ if($opt->{filter_field}) {
+ my @filt = grep /\S/, split /[\r\n]+/, $opt->{filter_field};
+ for(@filt) {
+ s/^\s+//;
+ s/\s+$//;
+ my ($t, $f) = split /\s*:\s*/, $_;
+ if(! $f) {
+ if ($opt->{multiple}) {
+ die "Must specify both table and filter for multiple table filters.\n";
+
+ }
+ else {
+ $f = $t;
+ $t = '';
+ }
+ $t ||= '';
+ }
+#::logDebug("found filter: t=$t f=$f");
+ my ($field, $filters) = split /\s*=\s*/, $f, 2;
+#::logDebug("found filter: t=$t field=$field filters=$filters");
+ $filter{$t}{$field} = $filters;
+ }
+ }
+
CONVERT: {
last CONVERT if ! $opt->{convert};
if ($opt->{convert} eq 'auto') {
@@ -128,6 +157,8 @@
$change_sub = sub {
my $table = shift;
$Vend::WriteDatabase{$table} = 1;
+ $Vend::TransactionDatabase{$table} = 1
+ if $opt->{transactions};
#::logDebug("changing table to $table");
$db = Vend::Data::database_exists_ref($table);
#::logDebug("db now=$db");
@@ -144,6 +175,8 @@
}
else {
$Vend::WriteDatabase{$table} = 1;
+ $Vend::TransactionDatabase{$table} = 1
+ if $opt->{transactions};
$db = Vend::Data::database_exists_ref($table);
die "Non-existent table '$table'\n" unless $db;
$db = $db->ref() unless $Vend::Interpolate::Db{$table};
@@ -163,8 +196,8 @@
# will get fields later
undef $opt->{fields};
}
- elsif($opt->{'fields'}) {
- $fields = $opt->{'fields'};
+ elsif($opt->{fields}) {
+ $fields = $opt->{fields};
$out .= "Using fields from parameter: '$fields'\n";
}
@@ -187,18 +220,65 @@
$fields =~ s/$delimiter/ /g;
$out .= "${tmsg}Using fields from file: '$fields'\n";
}
+ $filter{$table} ||= {};
die "No field names." if ! $fields;
my @names;
my $k;
my @f;
@names = split /\s+/, $fields;
my $key = shift @names;
- my @set;
my $i = 0;
my $idx = 0;
+
+ ######### Filters
+ ##
+ ## Done with so many data items for speed when empty....
+ ##
+
+ ## Holds filter subroutines if any
+ my %change;
+ ## Holds names of filter subroutines if any
+ my @filters;
+ ## Non-zero if found any filter
+ my $found_filter = 0;
+ ##
+ ######### Filters
+
+ # We skip the whole table if bad field is found
+ my $skipping;
for(@names) {
- $db->column_index($_);
- $set[$idx++] = $db->field_settor($_);
+ my $test = $db->column_index($_);
+#::logDebug("checking name=$_");
+ if(! defined $test) {
+ $out .= errmsg(
+ "Table %s: undefined column '%s'. Skipping table.",
+ $table,
+ $_,
+ );
+ $skipping = 1;
+ }
+ elsif ($filter{''}{$_} || $filter{$table}{$_}) {
+#::logDebug("found filter for name=$_");
+ my @things = grep length($_), $filter{''}{$_}, $filter{$table}{$_};
+ my $thing = join " ", @things;
+ eval {
+ $change{$_} = sub {
+ my $ref = shift;
+ $$ref = Vend::Interpolate::filter_value($thing, $$ref);
+ };
+ };
+ if($@) {
+ $out .= errmsg(
+ "Table %s: unrequited filter '%s'. Skipping table.",
+ $table,
+ $thing,
+ );
+ $skipping = 1;
+ }
+ push @filters, $_;
+ $found_filter++;
+ }
+ $idx++;
}
my %keys;
if ($opt->{cleanse}) {
@@ -219,9 +299,11 @@
$out .= "${tmsg}$delcount records deleted.\n" if $delcount;
$out .= "${tmsg}$addcount records added.\n" if $addcount;
$delcount = $totcount = $addcount = 0;
+ $db->commit() if $opt->{transactions};
$change_sub->($1);
redo TABLE;
}
+ next if $skipping;
if(! $k and ! length($k)) {
if ($f[0] eq 'DELETE') {
next if ! $opt->{delete};
@@ -257,6 +339,11 @@
if(@names) {
my %hash;
@hash{@names} = @f;
+ if($found_filter) {
+ for(@filters) {
+ $change{$_}->(\$hash{$_});
+ }
+ }
$db->set_slice($k, \%hash);
}
if($@) {
@@ -266,6 +353,9 @@
}
$count++;
}
+
+ $db->commit() if $opt->{transactions};
+
if ($opt->{cleanse}) {
# remove any record which hasn't updated
for (keys(%keys)) {
2.40 +23 -7 interchange/lib/Vend/Table/DBI.pm
rev 2.40, prev_rev 2.39
Index: DBI.pm
===================================================================
RCS file: /var/cvs/interchange/lib/Vend/Table/DBI.pm,v
retrieving revision 2.39
retrieving revision 2.40
diff -u -r2.39 -r2.40
--- DBI.pm 12 Jan 2003 06:47:09 -0000 2.39
+++ DBI.pm 12 Jan 2003 18:28:45 -0000 2.40
@@ -1,6 +1,6 @@
# Vend::Table::DBI - Access a table stored in an DBI/DBD database
#
-# $Id: DBI.pm,v 2.39 2003/01/12 06:47:09 jon Exp $
+# $Id: DBI.pm,v 2.40 2003/01/12 18:28:45 mheins Exp $
#
# Copyright (C) 1996-2002 Red Hat, Inc. <interchange@redhat.com>
#
@@ -20,7 +20,7 @@
# MA 02111-1307 USA.
package Vend::Table::DBI;
-$VERSION = substr(q$Revision: 2.39 $, 10);
+$VERSION = substr(q$Revision: 2.40 $, 10);
use strict;
@@ -216,6 +216,10 @@
$driver_name = $config->{BASE_CAPABILITY}
if $config->{BASE_CAPABILITY};
+ if($driver_name =~ /^dbi:+(.*?):/) {
+ $driver_name = $1;
+ }
+
my ($k, $known);
while ( ($k, $known) = each %known_capability ) {
if(! defined $config->{$k} ) {
@@ -288,6 +292,11 @@
sub create {
my ($class, $config, $columns, $tablename) = @_;
#::logDebug("trying create table $tablename");
+
+ local($config->{Transactions});
+ check_capability($config, $config->{DSN});
+ $config->{Transactions} = 1 if $config->{HAS_TRANSACTIONS};
+
my @call = find_dsn($config);
my $dattr = pop @call;
@@ -353,6 +362,7 @@
#::logDebug("Trying to create with specified CREATE_SQL:\n$config->{CREATE_SQL}");
eval {
$db->do($config->{CREATE_SQL});
+ $db->commit() if $config->{Transactions};
};
if($@) {
die ::errmsg(
@@ -370,11 +380,13 @@
$db->do("drop table $tablename")
and $config->{Clean_start} = 1
or warn "$DBI::errstr\n";
+ $db->commit() if $config->{Transactions};
};
#::logDebug("Trying to create with:$query");
eval {
$db->do($query);
+ $db->commit() if $config->{Transactions};
};
if($@) {
warn "DBI: Create table '$tablename' failed: $DBI::errstr\n";
@@ -382,8 +394,6 @@
else {
::logError("table %s created: %s" , $tablename, $query );
}
-
-
}
#::logDebug("seq: $config->{AUTO_SEQUENCE} create: $config->{SEQUENCE_CREATE}");
@@ -395,6 +405,7 @@
eval {
$db->do($dq)
or warn("drop sequence failed: $dq");
+ $db->commit() if $config->{Transactions};
};
}
$q =~ s/_SEQUENCE_NAME_/$config->{AUTO_SEQUENCE}/g;
@@ -406,6 +417,7 @@
eval {
$db->do($q)
or warn("create sequence failed: $q");
+ $db->commit() if $config->{Transactions};
};
}
@@ -438,6 +450,7 @@
$_,
$DBI::errstr,
);
+ $db->commit() if $config->{Transactions};
}
} elsif ($config->{AUTO_INDEX_PRIMARY_KEY}) {
# Oracle automatically creates indexes on primary keys,
@@ -445,6 +458,7 @@
} else {
$db->do("create index ${tablename}_${key} on $tablename ($key)")
or ::logError("table %s index failed: %s" , $tablename, $DBI::errstr);
+ $db->commit() if $config->{Transactions};
}
for(@index) {
@@ -455,6 +469,7 @@
$_,
$DBI::errstr,
);
+ $db->commit() if $config->{Transactions};
}
if(! defined $config->{EXTENDED}) {
@@ -752,10 +767,11 @@
sub quote {
my($s, $value, $field) = @_;
$s = $s->import_db() if ! defined $s->[$DBI];
- return 'NULL' if $field and ! length($value)
- and exists $s->[$CONFIG]->{PREFER_NULL}{$field};
+ return $s->[$DBI]->quote($value) unless $field;
+ return 'NULL' if ! length($value)
+ and exists $s->[$CONFIG]->{PREFER_NULL}{$field};
return $s->[$DBI]->quote($value)
- unless $field and exists $s->[$CONFIG]->{NUMERIC}{$field};
+ unless exists $s->[$CONFIG]->{NUMERIC}{$field};
$value = 0 if ! length($value);
return $value;
}
2.24 +3 -2 interchange/lib/Vend/Table/Common.pm
rev 2.24, prev_rev 2.23
Index: Common.pm
===================================================================
RCS file: /var/cvs/interchange/lib/Vend/Table/Common.pm,v
retrieving revision 2.23
retrieving revision 2.24
diff -u -r2.23 -r2.24
--- Common.pm 1 Jan 2003 14:54:09 -0000 2.23
+++ Common.pm 12 Jan 2003 18:28:45 -0000 2.24
@@ -1,6 +1,6 @@
# Vend::Table::Common - Common access methods for Interchange databases
#
-# $Id: Common.pm,v 2.23 2003/01/01 14:54:09 racke Exp $
+# $Id: Common.pm,v 2.24 2003/01/12 18:28:45 mheins Exp $
#
# Copyright (C) 1996-2002 Red Hat, Inc. <interchange@redhat.com>
# Copyright (C) 2003 ICDEVGROUP <interchange@icdevgroup.org>
@@ -23,7 +23,7 @@
# Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA.
-$VERSION = substr(q$Revision: 2.23 $, 10);
+$VERSION = substr(q$Revision: 2.24 $, 10);
use strict;
package Vend::Table::Common;
@@ -1325,6 +1325,7 @@
File::Copy::copy(@{$_});
}
}
+ $out->commit() if $out->config('HAS_TRANSACTIONS');
delete $out->[$CONFIG]{Clean_start};
delete $out->[$CONFIG]{_Dirty};
unlockfile(\*IN) or die "unlock\n";