Mike Gerwitz

Activist for User Freedom

aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMike Gerwitz <gerwitzm@lovullo.com>2017-07-12 15:21:43 -0400
committerMike Gerwitz <gerwitzm@lovullo.com>2017-07-12 15:58:37 -0400
commit2af0daceb320b59d13e312c6c9b65ae04e7a615b (patch)
tree4d7681fb1a0cea1a5bfa0c76dfbf9a0b7d5a71ec
parent52c1bb5ebe732582606116342730e76aaaba0c55 (diff)
downloadtame-2af0daceb320b59d13e312c6c9b65ae04e7a615b.tar.gz
tame-2af0daceb320b59d13e312c6c9b65ae04e7a615b.tar.bz2
tame-2af0daceb320b59d13e312c6c9b65ae04e7a615b.zip
Remove @keep support from linker
And everything else. This is a big (important) change; it addresses one of the greatest pains of the system. Keeps were added during the DSL rewrite (to support symbols and such) to work around the issue that there was no symbol-driven map; it allowed symbols to persist disjoint from the `__yield' dependency graph so that they could be mapped back and used by external systems. The problem with that is that it's both messy (coupling the concept of external dependencies with the actual code) and difficult to work with. It had a huge performance impact on the linker for two reasons: - Checking whether a package had already been seen and importing the keeps on first visit was expensive because of tree searching and manipulation; and - _every_ keep was imported and processed by the linker, even if it wouldn't end up being used by a particular program. The later especially had huge performance impacts on the entire system. The entire dependency graph is now map-driven, with the exception of the implicit `__yield' (which will eventually be moved into the map as well and the magic `lv:yield' removed in favor of a template). Performance-wise: our largest program ("dwelling") has many thousands of symbols and the largest package imported the majority of them, many of them unneeded, as the result of @keep subgraphs. Compilation of the largest package within that (for the UI) took about a minute and a half and ate up ~6GiB of RAM, for what really is a trivial task of resolving externs, some basic symbol processing, a topological sort, and ordering code fragments. After this change, it takes ~15s and less than 2GiB of RAM. Still a lot---and more improvements can be made---but much, much better. @keep and friends was left in rater.xsd so that nothing breaks while code is cleaned up; it'll be removed in the future. * src/current/compiler/linker.xsl: Remove @keep support. * src/current/dot/attr-keep.xsl: Remove now-unneeded template. * src/current/dot/defnode.xsl: Remove @keep and related. * src/current/include/preproc/eligclass.xsl: Remove @keep and related. * src/current/include/preproc/expand.xsl: Remove @keep and related. * src/current/include/preproc/macros.xsl: Remove @keep and related. * src/current/include/preproc/symtable.xsl: Remove @keep and related. * src/current/rater.xsd: Add TODO to remove @keep and friends.
-rw-r--r--src/current/compiler/linker.xsl184
-rw-r--r--src/current/dot/attr-keep.xsl50
-rw-r--r--src/current/dot/defnode.xsl1
-rw-r--r--src/current/include/preproc/eligclass.xsl5
-rw-r--r--src/current/include/preproc/expand.xsl5
-rw-r--r--src/current/include/preproc/macros.xsl4
-rw-r--r--src/current/include/preproc/symtable.xsl29
-rw-r--r--src/current/rater.xsd2
8 files changed, 24 insertions, 256 deletions
diff --git a/src/current/compiler/linker.xsl b/src/current/compiler/linker.xsl
index 23cef9a..93eb094 100644
--- a/src/current/compiler/linker.xsl
+++ b/src/current/compiler/linker.xsl
@@ -181,23 +181,21 @@
symbol, then it must not be used); note that lv:yield actually compiles
into a special symbol ___yield -->
<variable name="yields" as="element( preproc:sym )+">
- <copy-of select="preproc:sym[ @name='___yield' ]" />
+ <!-- TOOD: this shouldn't be a magical exception; map it -->
+ <sequence select="preproc:sym[ @name='___yield' ]" />
- <!-- also include anything derivable from any @keep symbol, either local
- or imported -->
- <copy-of select="preproc:sym[ @keep='true' ]" />
-
- <!-- TODO: these should be included as a consequence of the linking
- process, not as an exception -->
- <copy-of select="
+ <!-- TODO: messy; refactor this symbol situation -->
+ <!-- this should be the sole source of outputs and, consequently,
+ dependencies -->
+ <sequence select="
preproc:sym[
@type='map' or @type='map:head' or @type='map:tail'
or @type='retmap' or @type='retmap:head' or @type='retmap:tail'
]
" />
- <!-- TODO: same as above -->
- <copy-of select="preproc:sym[ @name='___worksheet' ]" />
+ <!-- TODO: also should not be an exception -->
+ <sequence select="preproc:sym[ @name='___worksheet' ]" />
</variable>
<!-- start at the top of the table and begin processing each symbol
@@ -325,81 +323,9 @@
</call-template>
</if>
- <variable name="pkg-seen" as="xs:boolean"
- select="(
- ( $cur/@src = '' or not( $cur/@src ) )
- and $stack/preproc:pkg-seen/@src = ''
- )
- or $cur/@src = $stack/preproc:pkg-seen/@src" />
-
- <variable name="newpending" as="element( l:pending )">
- <l:pending>
- <sequence select="$pending" />
-
- <!-- if this is the first time seeing this package, then pend its
- @keep's for processing -->
- <if test="not( $pkg-seen )">
- <message select="'[link] found package ', $cur/@src" />
-
- <variable name="document" as="element( lv:package )"
- select="if ( not( $cur/@src or $cur/@src = '' ) ) then
- $l:orig-root/lv:package
- else
- document( concat( $cur/@src, '.xmlo' ),
- $l:orig-root )
- /lv:package" />
-
- <variable name="keeps" as="element( preproc:sym )*" select="
- $document/preproc:symtable/preproc:sym[
- (
- @keep='true'
- or ( $l:orig-root/lv:package/@auto-keep-imports='true'
- and ( @type = 'class'
- or @type = 'cgen' ) )
- )
- and not(
- $l:orig-root/lv:package/@no-extclass-keeps='true'
- and @extclass='true'
- )
- and not( @name=$pending/@name )
- and not( @name=$stack/preproc:sym/@name )
- ]
- " />
-
- <variable name="keepdeps" as="element( preproc:sym )*">
- <call-template name="l:dep-aug">
- <with-param name="cur" select="$cur" />
- <with-param name="deps" select="$keeps" />
- <with-param name="proc-barrier" select="true()" />
- <with-param name="parent-name"
- select="concat( 'package ', $cur/@src )" />
- </call-template>
- </variable>
-
- <sequence select="$keepdeps" />
- </if>
- </l:pending>
- </variable>
-
-
- <variable name="stack-seen" as="element( l:sym-stack )">
- <l:sym-stack>
- <if test="not( $pkg-seen )">
- <sequence select="$stack/*" />
- <preproc:pkg-seen src="{$cur/@src}" />
- </if>
- </l:sym-stack>
- </variable>
-
- <variable name="newstack" as="element( l:sym-stack )"
- select="if ( $pkg-seen ) then
- $stack
- else
- $stack-seen" />
-
<apply-templates select="$cur" mode="l:depgen-process-sym">
- <with-param name="pending" select="$newpending/*" />
- <with-param name="stack" select="$newstack" />
+ <with-param name="pending" select="$pending" />
+ <with-param name="stack" select="$stack" />
<with-param name="path" select="$path" />
<with-param name="processing" select="
if ( $cur/@l:proc-barrier = 'true' )
@@ -610,11 +536,13 @@
select="$stack/preproc:sym[
@name=$cur/@name ]" />
+ <!-- TODO: this uses @name instead of @src because of map import
+ paths; decide on one or the other -->
<variable name="src-conflict" as="element( preproc:sym )*"
- select="if ( not( $cur/@src ) or $cur/@src = '' ) then
+ select="if ( not( $cur/@name ) or $cur/@name = '' ) then
()
else
- $existing[ not( @src = $cur/@src ) ]" />
+ $existing[ not( @name = $cur/@name ) ]" />
<if test="$src-conflict">
<call-template name="log:error">
@@ -622,7 +550,7 @@
<with-param name="msg">
<text>symbol name is not unique: `</text>
<value-of select="@name" />
- <text>' found in</text>
+ <text>' found in </text>
<value-of select="$cur/@src" />
<for-each select="$src-conflict">
@@ -805,20 +733,13 @@
<!-- TODO: some better way. -->
-<template match="preproc:sym[ starts-with( @type, 'map' ) or starts-with( @type, 'retmap' ) ]"
+<template match="preproc:sym[ starts-with( @type, 'map' ) ]"
mode="l:depgen-sym" priority="7">
<!-- do not process deps -->
</template>
-<template mode="l:depgen-sym" as="element()*"
- match="preproc:pkg-seen"
- priority="5">
- <sequence select="." />
-</template>
-
-
<template mode="l:depgen-sym" as="element( preproc:sym )*"
match="preproc:sym[
@type='const' ]"
@@ -1384,78 +1305,7 @@
<template match="preproc:sym" mode="l:map" priority="5">
- <param name="symtable" as="element( l:dep )" />
- <param name="type" as="xs:string"
- select="'input'" />
- <param name="from" as="xs:string"
- select="'destination'" />
- <param name="ignore-error" as="xs:boolean"
- select="false()" />
-
- <variable name="name" as="xs:string"
- select="@name" />
- <variable name="src" as="xs:string"
- select="@src" />
-
- <!-- map symbols must always be remote -->
- <variable name="pkg" as="element( lv:package )"
- select="document( concat( @src, '.xmlo' ), . )
- /lv:package" />
-
- <!-- get map symbol dependencies -->
- <variable name="deps" as="element( preproc:sym-dep )*"
- select="$pkg/preproc:sym-deps/
- preproc:sym-dep[ @name=$name ]" />
-
- <if test="not( $deps )">
- <call-template name="log:internal-error">
- <with-param name="name" select="'link'" />
- <with-param name="msg">
- <text>could not locate symbol dependencies: </text>
- <value-of select="concat( @src, '/', @name )" />
- </with-param>
- </call-template>
- </if>
-
- <!-- FIXME: we should not have to check for @yields here; we may
- have to require imports in the map to satisfy normalization
- before-hand -->
- <variable name="unknown" as="element( preproc:sym-ref )*"
- select="$deps/preproc:sym-ref[
- not( @name=$symtable/preproc:sym/@name
- or @name=$symtable/preproc:sym/@yields ) ]" />
-
- <choose>
- <!-- ensure that every dependency is known (we only care that the symbol
- actually exists and is an input) -->
- <when test="$unknown and not( $ignore-error )">
- <for-each select="$unknown">
- <call-template name="log:error">
- <with-param name="terminate" select="'no'" />
- <with-param name="name" select="'link'" />
- <with-param name="msg">
- <value-of select="$type" />
- <text> map </text>
- <value-of select="$from" />
- <text> </text>
- <value-of select="@name" />
- <text> is not a known </text>
- <value-of select="$type" />
- <text> field for </text>
- <value-of select="concat( $src, '/', $name )" />
- <text>; ensure that it exists and is either used or has @keep set</text>
- </with-param>
- </call-template>
- </for-each>
-
- <l:map-error />
- </when>
-
- <!-- good to go; link symbol -->
- <otherwise>
- <apply-templates select="." mode="l:link-deps" />
- </otherwise>
- </choose>
+ <apply-templates select="." mode="l:link-deps" />
</template>
</stylesheet>
diff --git a/src/current/dot/attr-keep.xsl b/src/current/dot/attr-keep.xsl
deleted file mode 100644
index d277674..0000000
--- a/src/current/dot/attr-keep.xsl
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
- Styles node based on keep flag
-
- Copyright (C) 2016 LoVullo Associates, Inc.
-
- This file is part of TAME.
-
- TAME is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see
- <http://www.gnu.org/licenses/>.
--->
-<xsl:stylesheet version="2.0"
- xmlns="http://www.w3.org/1999/xhtml"
- xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-
- xmlns:dot="http://www.lovullo.com/calc/dot"
- xmlns:preproc="http://www.lovullo.com/rater/preproc">
-
-
-<!--
- External nodes should be styled as such
--->
-<xsl:template mode="dot:attr-keep" priority="5" match="
- *[ @keep='true' ]
- ">
-
- <dot:attr name="fontcolor">red</dot:attr>
-</xsl:template>
-
-
-
-<!--
- Default node attributes
--->
-<xsl:template match="preproc:sym" mode="dot:defnode-keep" priority="1">
-</xsl:template>
-
-</xsl:stylesheet>
-
diff --git a/src/current/dot/defnode.xsl b/src/current/dot/defnode.xsl
index 1cf6501..6e350df 100644
--- a/src/current/dot/defnode.xsl
+++ b/src/current/dot/defnode.xsl
@@ -92,7 +92,6 @@
<xsl:apply-templates select="." mode="dot:attr-extern" />
<xsl:apply-templates select="." mode="dot:attr-color" />
<xsl:apply-templates select="." mode="dot:attr-shape" />
- <xsl:apply-templates select="." mode="dot:attr-keep" />
</xsl:with-param>
</xsl:call-template>
</xsl:variable>
diff --git a/src/current/include/preproc/eligclass.xsl b/src/current/include/preproc/eligclass.xsl
index 5fc9353..974fb7a 100644
--- a/src/current/include/preproc/eligclass.xsl
+++ b/src/current/include/preproc/eligclass.xsl
@@ -101,11 +101,6 @@
<lv:classify as="{$as}" yields="{$yields}"
desc="{@name} package is eligible">
- <!-- TODO: this should really be a compile-time value -->
- <xsl:if test="@keep-elig-class = 'true'">
- <xsl:attribute name="keep" select="'true'" />
- </xsl:if>
-
<!-- each of our imported packages' elig classes must be truthful -->
<xsl:apply-templates mode="preproc:gen-elig-class-matches"
select="lv:import">
diff --git a/src/current/include/preproc/expand.xsl b/src/current/include/preproc/expand.xsl
index c8b2459..f023df1 100644
--- a/src/current/include/preproc/expand.xsl
+++ b/src/current/include/preproc/expand.xsl
@@ -500,11 +500,6 @@
<xsl:apply-templates mode="preproc:expand"
select="@*" />
- <!-- force @keep on @terminate -->
- <xsl:if test="@terminate='true'">
- <xsl:attribute name="keep" select="'true'" />
- </xsl:if>
-
<!-- copy everything else -->
<xsl:apply-templates mode="preproc:expand" />
</xsl:copy>
diff --git a/src/current/include/preproc/macros.xsl b/src/current/include/preproc/macros.xsl
index dd836c9..8d991a4 100644
--- a/src/current/include/preproc/macros.xsl
+++ b/src/current/include/preproc/macros.xsl
@@ -461,9 +461,7 @@
<!-- ensure that this name does not already exist -->
<xsl:if test="not( /lv:*/lv:classify[ @as=$genas ] )">
- <!-- TODO: We're flagging as @keep for now due to gclass needs, but this
- should be removed later -->
- <lv:classify as="{$genas}" desc="{@desc}" keep="true">
+ <lv:classify as="{$genas}" desc="{@desc}">
<lv:match on="{@name}" value="TRUE" />
</lv:classify>
</xsl:if>
diff --git a/src/current/include/preproc/symtable.xsl b/src/current/include/preproc/symtable.xsl
index e9a7c61..432c3ab 100644
--- a/src/current/include/preproc/symtable.xsl
+++ b/src/current/include/preproc/symtable.xsl
@@ -517,7 +517,6 @@
<xsl:param name="orig-root" />
<xsl:param name="package" select="@package" />
<xsl:param name="export" select="@export" />
- <xsl:param name="ignore-keep" select="@ignore-keep" />
<xsl:param name="no-extclass" select="@no-extclass" />
<xsl:param name="keep-classes" select="@keep-classes" />
@@ -593,7 +592,7 @@
<xsl:when test="@pollute='true'
and @local='true'
and not( @extern='true' )">
- <xsl:sequence select="@name, @src, @pollute, @keep, @parent, @extclass" />
+ <xsl:sequence select="@name, @src, @pollute, @parent, @extclass" />
</xsl:when>
<!-- copy all the symbol information -->
@@ -633,23 +632,6 @@
</xsl:choose>
</xsl:attribute>
- <!-- keep manipulation: *always* keep classes if requested, even if
- @ignore-keep is provided; in the case of the latter, unsets @keep -->
- <xsl:choose>
- <!-- keep classes when requested -->
- <xsl:when test="
- ( @type = 'class' or @type = 'cgen' )
- and $keep-classes = 'true'">
-
- <xsl:attribute name="keep" select="'false'" />
- </xsl:when>
-
- <!-- demolish @keep if requested -->
- <xsl:when test="$ignore-keep = 'true'">
- <xsl:attribute name="keep" select="'false'" />
- </xsl:when>
- </xsl:choose>
-
<!-- children should always be copied, unless poluting -->
<xsl:if test="not( @pollute='true' and @local='true' )">
<xsl:sequence select="preproc:*" />
@@ -663,7 +645,7 @@
<xsl:variable name="external" select="boolean( @external='true' )" />
<preproc:sym name="{@yields}" type="rate"
- extclass="{$external}" keep="{boolean( @keep )}"
+ extclass="{$external}"
local="{@local}" dtype="float" dim="0" tex="{@sym}" />
<xsl:apply-templates mode="preproc:symtable" />
@@ -706,7 +688,7 @@
<xsl:template match="c:*[ @generates ]" mode="preproc:symtable" priority="5">
<xsl:variable name="parent" select="ancestor::lv:rate" />
- <preproc:sym name="{@generates}" keep="{boolean( $parent/@keep )}"
+ <preproc:sym name="{@generates}"
parent="{$parent/@yields}"
type="gen" dtype="float" dim="1" desc="{@desc}" tex="{@sym}" />
@@ -718,10 +700,9 @@
<xsl:template match="lv:classify" mode="preproc:symtable" priority="5">
<xsl:variable name="external" select="boolean( @external='true' )" />
<xsl:variable name="terminate" select="boolean( @terminate='true' )" />
- <xsl:variable name="keep" select="boolean( @keep='true' )" />
<preproc:sym name=":class:{@as}"
- extclass="{$external}" terminate="{$terminate}" keep="{$keep}"
+ extclass="{$external}" terminate="{$terminate}"
type="class" dim="?" desc="{@desc}" yields="{@yields}"
orig-name="{@as}">
@@ -733,7 +714,7 @@
to avoid scanning separate object files for such common information)
-->
<xsl:if test="@yields">
- <preproc:sym name="{@yields}" keep="{$keep}"
+ <preproc:sym name="{@yields}"
parent=":class:{@as}"
extclass="{$external}" terminate="{$terminate}"
type="cgen" dtype="boolean" dim="?" desc="{@desc}">
diff --git a/src/current/rater.xsd b/src/current/rater.xsd
index d1bac47..f27d75a 100644
--- a/src/current/rater.xsd
+++ b/src/current/rater.xsd
@@ -1,5 +1,5 @@
<?xml version="1.0"?>
-
+<!-- TODO: Remove @keep -->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
targetNamespace="http://www.lovullo.com/rater"
xmlns="http://www.lovullo.com/rater"