Item7616: Control over formfield rendering in a formatted SEARCH; fix performance issue with formatted SEARCH in TWiki-6.0

TWiki:Codev.ControlOverFormFieldRenderingInSearch

-- TWiki:Main/PeterThoeny - 2015-02-25

Performance measurement on TWiki::Search using Monitor on a complex page with 5 SEARCHes:

To get the improved performance in your TWiki-6.0.x search (up to 3x!), best to get the latest Search.pm from:
http://svn.twiki.org/svn/twiki/branches/TWikiRelease06x00/core/lib/TWiki/Search.pm

Alternatively, apply the following patch:

[pthoeny@twiki lib]$ svn diff -r 28748:28829 TWiki/Search.pm
Index: TWiki/Search.pm
===================================================================
--- TWiki/Search.pm   (revision 28748)
+++ TWiki/Search.pm   (revision 28829)
@@ -35,6 +35,9 @@
 require TWiki::Sandbox;
 require TWiki::Render; # SMELL: expensive
 
+#use Monitor;
+#Monitor::MonitorMethod('TWiki::Search');
+
 my $queryParser;
 
 BEGIN {
@@ -725,7 +728,8 @@
                     # removed elements are saved for later use
             }
 
-            $topicInfo = _sortTopics( $this, $web, \@topicList, $sortOrder, $revSort );
+            # sort topics and store topic info in $topicInfo
+            _sortTopics( $this, $topicInfo, $web, \@topicList, $sortOrder, $revSort );
             if ( $start ne '' && @saveShortened ) {
                 # for pagination, which is indicated by the presense of the
                 # =start= parameter, the removed elements for optimization are
@@ -734,8 +738,8 @@
             }
         }
         elsif ( $sortOrder =~ /\b(created?|editby|formfield\((.*?)\)|parent(\(.*?\))?)([, ]|$)/ ) {
-            # sort by topic creation time, author, parent
-            $topicInfo = _sortTopics( $this, $web, \@topicList, $sortOrder, $revSort );
+            # sort by topic creation time, author, parent, and store topic info in $topicInfo
+            _sortTopics( $this, $topicInfo, $web, \@topicList, $sortOrder, $revSort );
         }
         else {
 
@@ -812,7 +816,7 @@
             my $forceRendering = 0;
             unless ( exists( $topicInfo->{$topic} ) ) {
                 # not previously cached
-                $topicInfo->{$topic} = _extractTopicInfo( $this, $web, $topic, 0 );
+                $topicInfo->{$topic} = _extractTopicInfo( $this, $topicInfo, $web, $topic, 0 );
             }
             my $epochSecs = $topicInfo->{$topic}->{modified};
             require TWiki::Time;
@@ -970,6 +974,16 @@
 s/\$formfield\(\s*([^\)]*)\s*\)/displayFormField( $meta, $1 )/ges;
                     $out =~
 s/\$parent\(([^\)]*)\)/TWiki::Render::breakName( $meta->getParent(), $1 )/ges;
+                    # undocumented $breadcrumb returning list with "parents, topic",
+                    # where parents is parent breadcrumb (only if sort by parent(...))
+                    $out =~ s/\$breadcrumb/
+                        if( $topicInfo->{$topic}{parent} ) {
+                            $topicInfo->{$topic}{parent};
+                        } else {
+                            my $p = $meta->getParent();
+                            ( $p ? "$p, $topic" : $topic );
+                        }
+                      /ges;
                     $out =~ s/\$parent/$meta->getParent()/ges;
                     $out =~ s/\$formname/$meta->getFormName()/ges;
                     $out =~ s/\$count\((.*?\s*\.\*)\)/_countPattern( $text, $1 )/ges;
@@ -1131,15 +1145,18 @@
     return $searchResult;
 }
 
+# RE for a full-spec floating-point number
+my $number = qr/^[-+]?[0-9]+(\.[0-9]*)?([Ee][-+]?[0-9]+)?$/s;
+
 # extract topic info required for sorting and sort.
 sub _sortTopics {
-    my ( $this, $web, $topics, $sortfield, $revSort ) = @_;
+    my ( $this, $topicInfo, $web, $topics, $sortfield, $revSort ) = @_;
 
     my $users = $this->{session}->{users};
-    my $topicInfo = {};
     my $topicParents = {}; # initialize parent hash, used to optimize sorting by parents
     foreach my $topic ( @$topics ) {
-        $topicInfo->{$topic} = _extractTopicInfo( $this, $web, $topic, $sortfield, $topicParents );
+        $topicInfo->{$topic} = _extractTopicInfo( $this, $topicInfo, $web, $topic,
+                                                  $sortfield, $topicParents );
     }
 
     $sortfield =~ s/\bformfield\((.*?)\)/$1/g;
@@ -1177,7 +1194,19 @@
     foreach my $sortToken ( @sortTokens ) {
         @$topics =
           map { $_->[1] }
-          sort { _compare( $b->[0], $a->[0] ) }
+          sort {
+              $_ = 0;
+              if( defined $a->[0] && defined $b->[0] ) {
+                  if( $a->[0] =~ /$number/o && $b->[0] =~ /$number/o ) {
+                      # when sorting numbers do it largest first; this is just because
+                      # this is what date comparisons need.
+                      $_ = $a->[0] <=> $b->[0];
+                  } else {
+                      $_ = $a->[0] cmp $b->[0];
+                  }
+              }
+              $_;
+          }
           map { [ $topicInfo->{$_}->{$sortToken}, $_ ] }
           @$topics;
         @$topics = reverse @$topics if( $reverseTokens[$i++] );
@@ -1186,31 +1215,15 @@
     return $topicInfo;
 }
 
-# RE for a full-spec floating-point number
-my $number = qr/^[-+]?[0-9]+(\.[0-9]*)?([Ee][-+]?[0-9]+)?$/s;
-
-sub _compare {
-    return 0 unless( defined $_[0] && defined $_[1] );
-    if( $_[0] =~ /$number/o && $_[1] =~ /$number/o ) {
-
-        # when sorting numbers do it largest first; this is just because
-        # this is what date comparisons need.
-        return $_[1] <=> $_[0];
-    }
-    else {
-        return $_[1] cmp $_[0];
-    }
-}
-
 # extract topic info
 sub _extractTopicInfo {
-    my ( $this, $web, $topic, $sortfield, $topicParents ) = @_;
+    my ( $this, $topicInfo, $web, $topic, $sortfield, $topicParents ) = @_;
     my $info    = {};
     my $session = $this->{session};
     my $store   = $session->{store};
     my $users   = $this->{session}->{users};
 
-    my ( $meta, $text ) = _getTextAndMeta( $this, undef, $web, $topic );
+    my ( $meta, $text ) = _getTextAndMeta( $this, $topicInfo, $web, $topic );
 
     $info->{text} = $text;
     $info->{meta} = $meta;
@@ -1234,7 +1247,7 @@
         # sort by parent breadcrumb up to indicated level.
         # for example, sorting on 3 levels is done with string "GrandParent, Parent, Topic"
         my $level = $2 || 1;
-        my @parents = ();
+        my @parents = ( $topic );
         my $parent = $meta->getParent();
         $parent =~ s/.*\.//; # cut web prefix if present
         while( $level-- >= 1 && $parent ) {
@@ -1245,11 +1258,12 @@
                 if( $topicParents && $topicParents->{$parent} ) {
                     $gParent = $topicParents->{$parent};
                 } elsif( $store->topicExists( $web, $parent ) ) {
-                    my ( $gpMeta ) = _getTextAndMeta( $this, undef, $web, $parent );
+                    my ( $gpMeta ) = _getTextAndMeta( $this, $topicInfo, $web, $parent );
                     $gParent = $gpMeta->getParent();
                     $gParent =~ s/.*\.//; # cut web prefix if present
                 }
                 $gParent = '' if( $gParent eq $parent ); # stop if topic points to itself as parent
+                $topic = $parent;
                 $parent = $gParent || '';
             }
         };
@@ -1265,10 +1279,14 @@
 sub _setFormFieldInfo {
     my ( $info, $meta, $formfield ) = @_;
     unless ( defined( $info->{$formfield} ) ) {
-        $info->{$formfield} = displayFormField( $meta, $formfield );
+        $info->{$formfield} = displayFormField( $meta, $formfield, 1 );
     }
 }
 
+# pre-compile regex
+my $reATTACHURL     = qr/%ATTACHURL%/;
+my $reATTACHURLPATH = qr/%ATTACHURLPATH%/;
+
 # get the text and meta for a topic
 sub _getTextAndMeta {
     my ( $this, $topicInfo, $web, $topic ) = @_;
@@ -1285,12 +1303,10 @@
         ( $meta, $text ) = $store->readTopic( undef, $web, $topic, undef );
         $text =~ s/%WEB%/$web/gos;
         $text =~ s/%TOPIC%/$topic/gos;
-        my $reATTACHURL = qr/%ATTACHURL%/;
         if ( $text =~ $reATTACHURL ) {
             my $attachUrl = $this->{session}->getPubUrl(1, $web, $topic);
             $text =~ s/$reATTACHURL/$attachUrl/gos;
         }
-        my $reATTACHURLPATH = qr/%ATTACHURLPATH%/;
         if ( $text =~ $reATTACHURLPATH ) {
             my $attachUrlPath = $this->{session}->getPubUrl(0, $web, $topic);
             $text =~ s/$reATTACHURLPATH/$attachUrlPath/gos;
@@ -1366,25 +1382,51 @@
 =cut
 
 sub displayFormField {
-    my( $meta, $args ) = @_;
+    my( $meta, $args, $skipRendering ) = @_;
 
-    my $encodeType = '';
-    if ( $args =~ s/\s*,\s*encode:(\w+)// ) {
-        $encodeType = $1;
+    my $render = '';
+    if( $args =~ s/\s*,\s*render:(\w+)// ) {
+        $render = $1;
     }
+    my $attrs = { protectdollar => 1, showhidden => 1 };
+    if( $args =~ s/\s*,\s*encode:(\w+)// ) {
+        $attrs->{encode} = $1;
+    }
     my $name = $args;
-    my $breakArgs = '';
-    my @params = split( /\,\s*/, $args, 2 );
-    if( @params > 1 ) {
-        $name = $params[0] || '';
-        $breakArgs = $params[1] || 1;
+    if( $name =~ /\,/ ) {
+        my @params = split( /\,\s*/, $name, 2 );
+        if( @params > 1 ) {
+            $name = $params[0] || '';
+            $attrs->{break} = $params[1] || 1;
+        }
     }
+    return '' unless $name;
 
-    return $meta->renderFormFieldForDisplay(
-        $name, '$value',
-        { break => $breakArgs, protectdollar => 1, showhidden => 1,
-          encode => $encodeType, }
-    );
+    # Item7616: Reverting partly to TWiki-5.1's Item6082 fix for performance.
+    # $meta->renderFormFieldForDisplay is slow, doubling the time of a SEARCH
+    # with formfields. Let's avoid it where feasible, e.g. avoid in sorting and
+    # if no formatting needed.
+    if( $skipRendering || ! ( $render eq 'display' || $attrs->{break} || $attrs->{encode} ) ) {
+        my $form =  $meta->get( 'FORM' );
+        my $fields;
+        if( $form ) {
+            $fields = $meta->get( 'FIELD', $name );
+            unless( $fields ) {
+                # not a valid field name, maybe it's a title.
+                require TWiki::Form;
+                $fields = $meta->get( 'FIELD', TWiki::Form::fieldTitle2FieldName( $name ) );
+            }
+        }
+        if( ref( $fields ) eq 'HASH' ){
+            # fix for Item6167, this line was not added for fixing Item6082
+            my $val = $fields->{value};
+            $val =~ s/\$(n|nop|quot|percnt|dollar)/\$<nop>$1/g;
+            return $val;
+        }
+        return ''; # form field not found
+    }
+
+    return $meta->renderFormFieldForDisplay( $name, '$value', $attrs );
 }
 
 # Returns the topic revision info of the base version,

-- TWiki:Main.PeterThoeny - 2015-03-04

Re-opening this because this patch has a small bug: $formfield(Name By Title) did not work.

-- TWiki:Main.PeterThoeny - 2015-03-24

This is now fixed. I updated above patch.

-- TWiki:Main.PeterThoeny - 2015-03-24