#] #] ********************* #] "$d_Qtest"'symbols flatliner derivations/symbols system loaddefs- develop.txt' - setup symbol system # www.BillHowell.ca 02Aug2018 initial # view this file in a text editor, with [constant width font, tab = 3 spaces], no line-wrap 24************************24 # Table of Contents : # $ grep "^#]" "$d_Qtest"'symbols flatliner derivations/symbols system loaddefs- develop.txt' | sed 's/^#\]/ /' ********************* "$d_Qtest"'symbols flatliner derivations/symbols system loaddefs- develop.txt' - setup symbol system ??Jul or Aug2018 I split the output to try to capture proper unicode : doesn't work : 01Aug2018 sym_stdForm_file problem with - "('=<>#$%' EACHLEFT in str)" 01Aug2018 Attempts to resolve unicode mashing 02Aug2018 fix unicode mashing 13Nov2018 Restart work after 2 months on IJCNN2018 stuff 22Nov2018 sym_stdForm 22Nov2018 strings_alltest results and fixes - reset code back to early September when it worked 24************************24 08********08 #] ??Jul or Aug2018 I split the output to try to capture proper unicode : doesn't work : +-----+ sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL finn fout ftmp i_line i_line_old str uniStr uniStr_old ; % ; pname_inn := link d_inn fname_base '.txt' ; pname_tmp := link d_out fname_base ' tmp.txt' ; pname_out := link d_out fname_base ' stdForm.txt' ; % ; finn := open pname_inn "r ; ftmp := open pname_tmp "w ; i_line := 0 ; i_line_old := 1 ; uniStr_old := null ; WHILE (~= ??eof (str := readfile finn)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=#<>$%' EACHLEFT in str)) THEN %write uniStr ; chr1 := str@0 ; IF (= chr_tab chr1) THEN chr1 := ' ' ; ELSE chr1 := string chr1 ; ENDIF ; writefile ftmp (link chr1 ' ' (string i_line_old) ' ' uniStr_old ' ') ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; %writefile ftmp (link chr1 ' ' (string i_line_old) ' ' uniStr_old ' ') ; EACH close finn ftmp ; % ; ftmp := open pname_tmp "r ; fout := open pname_out "w ; WHILE (~= ??eof (str := readfile ftmp)) DO writefile fout str ; ENDWHILE ; EACH close ftmp fout ; } +-----+ #+-----+ # old code writefile fot (link link link (string str@0) ' ' (string i_line_old) ' ' chrList ' ') ; chrList := null ; uniList := str_to_unicodeList str ; FOR i WITH (tell gage shape uniList) DO chrList := link chrList uniList@i ; ENDFOR ; uniStr := str_stdFormUnicode str ; sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL in fot i_line i_line_old str uniStr uniStr_old ; IF flag_break THEN BREAK ; ENDIF ; pname_inn := link d_inn fname_base '.txt' ; pname_out := link d_out fname_base ' stdForm.txt' ; fin := open pname_inn "r ; fot := open pname_out "w ; i_line := 0 ; i_line_old := 1 ; % ; WHILE (~= ??eof (str := readfile fin)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=<>#$%' EACHLEFT in str)) THEN write uniStr ; writefile fot (link (string str@0) ' ' (string i_line_old) ' ') ; writefile fot uniStr_old ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; writefile fot (link link 'eof ' (string i_line_old) ' ' uniStr_old ' ') ; EACH close fin fot ; } very strange - I had to bye, re-loadefs, but if I edited that line the error recurred!??? crazy - I copied the same line from "symbols extract.ndf", which does not cause a problem, but sym_stdForm_file fails??!!! I bracketed problem line with : % ; >> same problem! Using "pass" instead of "in" works - why? I may have to reboot? 08********08 #] 01Aug2018 sym_stdForm_file problem with - "('=<>#$%' EACHLEFT in str)" sym_stdForm_file moved to "symbols system loaddefs.ndf" Maybe EACHLEFT isn't legal within an IF clause? (aounds nuts) Try : flag_line_start := OR ('=<>#$%' EACHLEFT in str) ; IF flag_line_start THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ) ; IF +-----+ >> nyet Try : flag_line_start := '=<>#$%' EACHLEFT in str ; IF OR flag_line_start THEN +-----+ qnial> str := 'Hello =<>#$% test' Hello =<>#$% test qnial> '=<>#$%' EACHLEFT in str llllll qnial> '=<>#$`%' EACHLEFT in str lllllol It seems to be the variable "sym" that is causing the problem, but WHY? How to fix? Try : +-----+ code str := null ; % ; WHILE (~= ??eof (str := readfile fin)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; flag_line_start := '=<>#$%' EACHLEFT in str ; write 'str= ' str ; IF OR flag_line_start THEN write uniStr ; writefile fot (link (string str@0) ' ' (string i_line_old) ' ') ; writefile fot uniStr_old ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; +-----+ response qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ; WRITE 'str= ' NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" <<<<<< loading ended : symbols system loaddefs.ndf errors found: 1 +-----+ But a quick example works : +-----+qnial> str := 'Hello =<>#$% test' Hello =<>#$% test qnial> '=<>#$%' EACHLEFT in str llllll +-----+ >> so the variable "str" somehow is a problem Call it "strr" +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STRR <***> ; WRITE 'str= ' +-----+ >> nyet Not the variable name. Define a test operator : +-----+ sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL in fot i_line i_line_old str uniStr uniStr_old ; IF flag_break THEN BREAK ; ENDIF ; i_line := 0 ; i_line_old := 1 ; str := 'Hello =<>#$% test' ; flag_line_start := '=<>#$%' EACHLEFT in str ; write flag_line_start ; } +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ; WRITE FLAG_LINE_START >> NYET Remove flag_break line : +-----+ sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL in fot i_line i_line_old str uniStr uniStr_old ; i_line := 0 ; i_line_old := 1 ; str := 'Hello =<>#$% test' ; flag_line_start := '=<>#$%' EACHLEFT in str ; write flag_line_start ; } +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ; WRITE FLAG_LINE_START >> Same Go simpler yet : +-----+ sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL in fot i_line i_line_old str uniStr uniStr_old ; i_line := 0 ; i_line_old := 1 ; str := 'Hello =<>#$% test' ; flag_line_start := `= in str ; write flag_line_start ; } +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" <<<<<< loading ended : symbols system loaddefs.ndf +-----+ >> Bingo. So why am I having problems in loaddefs with "EACHLEFT in" ?? Try integers : +-----+ str := null ; % ; WHILE (~= ??eof (str := tell 20)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; flag_line_start := 4 9 100 EACHLEFT in str ; write 'str= ' str ; IF OR flag_line_start THEN write uniStr ; writefile fot (link (string str@0) ' ' (string i_line_old) ' ') ; writefile fot uniStr_old ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" <<<<<< loading ended : symbols system loaddefs.ndf +-----+ >> Nice, no problem Guesses : - `% interferes with the interpreter (except that is not a problem in other operator WHILE loops!) - the particular sequence is the problem Back to earlier form, without '<>' : +-----+ WHILE (~= ??eof (str := readfile fin)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; write 'str= ' str ; IF (OR ('=#$%' EACHLEFT in str)) THEN write uniStr ; writefile fot (link (string str@0) ' ' (string i_line_old) ' ') ; writefile fot uniStr_old ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" <<<<<< loading ended : symbols system loaddefs.ndf +-----+ >> SAME PROBLEM!!!??? I NEED to detect '<>' characters to pick out multi-line expressions!! Try : +-----+ IF (OR ('abc' EACHLEFT in str)) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ) ) THEN +-----+ >> crap Get rid of OR +-----+ IF ('abc' EACHLEFT in str) THEN +-----+ >> same problem The following sequence of examples are illustrative : +-----+ IF (OR ('=#$%' EACHLEFT in str)) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ) ) THEN +-----+ +-----+ IF (OR (pname_inn EACHLEFT in str)) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ) ) THEN +-----+ +-----+ IF ('=#$%' EACHLEFT in str) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf ?unexpected token or eol: EACHLEFT IN STR <***> ) THEN WRITE +-----+ +-----+ IF (OR ('=#$%' EACHLEFT pass str)) THEN +-----+ +qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" +-----+ +-----+ IF (OR ('=#$%' EACHLEFT = str)) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" +-----+ +-----+ IF (= i_line_old str) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" +-----+ +-----+ IF (i_line_old EACHLEFT = str) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" +-----+ This really does seem like an interpreter bug, but I would have to check very closely in the language specs. 01Aug2018 12:02 YIKES!!!!!!!!!!!!!!!!!!! ?? !!!!!!!!!!!!!!!!!!!!!!!!!!!! I defined "in" as LOCAL!!!! IDIOT!!!! NOW it works!!!! +-----+ IF (OR ('=#$%' EACHLEFT in str)) THEN +-----+ qnial> loaddefs '/media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf' >>>>>> loading start : symbols system loaddefs.ndf NOTE : for tests of symbol [strings, extract,translate], type "lq_symTests" +-----+ 08********08 #] 01Aug2018 Attempts to resolve unicode mashing # Approach : dual-file output sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL finn fout i_line i_line_old str uniStr uniStr_old ; % ; pname_inn := link d_inn fname_base '.txt' ; pname_t1 := link d_out fname_base ' n_lines.txt' ; pname_t2 := link d_out fname_base ' expresn.txt' ; pname_out := link d_out fname_base ' stdForm.txt' ; % ; finn := open pname_inn "r ; f_1 := open pname_f1 "w ; f_2 := open pname_t2 "w ; i_line := 0 ; i_line_old := 1 ; uniStr_old := null ; WHILE (~= ??eof (str := readfile finn)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=#<>$%' EACHLEFT in str)) THEN writefile f_1 (string str@0) ; writefile f_2 (link (string i_line_old) ' ' (link uniStr_old) ' ') ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; i_line_old := i_line_old + 1 ; writefile fout (link (string i_line_old) ' ' (link uniStr_old) ' ') ; EACH close finn f_1 f_2 ; % ; f_1 := open pname_f1 "r ; f_2 := open pname_t2 "r ; fout := open pname_out "w ; WHILE (~= ??eof (str := readfile f_2)) DO chr1 := readfile f_1 ; i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=#<>$%' EACHLEFT in str)) THEN writefile f_1 (string str@0) ; writefile f_2 (link (string i_line_old) ' ' (link uniStr_old) ' ') ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; EACH close f_1 f_2 fout ; } # search-replace end : fot # bad unicode % no output ; writefile fout (link (string i_line_old) ' ' uniStr_old@0 ' ' uniStr_old ' ') ; % mashed unicode ; writefile ftmp (link (string str@0) ' ' (string i_line_old) ' ' (link uniStr_old) ' ') ; ... chr1 := str@0 ; IF (= chr_tab chr1) THEN chr1 := ' ' ; ENDIF ; writefile ftmp (link chr1 ' ' (string i_line_old) ' ' (link uniStr_old) ' ') ; ... chr1 := str@0 ; IF (= chr_tab chr1) THEN chr1 := ' ' ; ENDIF ; writefile ftmp (link chr1 ' ' (string i_line_old) ' ' uniStr_old ' ') ; ... chr1 := str@0 ; IF (= chr_tab chr1) THEN chr1 := ' ' ; ELSE chr1 := string chr1 ; ENDIF ; writefile fout (link chr1 ' ' (string i_line_old) ' ' uniStr_old ' ') ; ... this "should" have succeeded, and while "expresn" is OK, "stdForm" is mashed. Why does [link,append] ruin the code? chr1 := str@0 ; IF (= chr_tab chr1) THEN chr1 := ' ' ; ELSE chr1 := link chr1 ' ' ; ENDIF ; writefile f_1 (link chr1 (string i_line_old)) ; writefile f_2 (link uniStr_old) ; # good "symParen_file_bads test stdForm.txt", nut no "symParen_file_bads test stdForm HFLN.txt" & downstream as the line start codes are lost : writefile ftmp (link (string i_line_old) ' ' (link uniStr_old) ' ') ; # good unicode : writefile ftmp (link uniStr_old) ; writefile ftmp (link (string i_line_old) ' ' (link uniStr_old) ' ') ; # not very helpful : pname_t1 := link d_out fname_base ' n_lines.txt' ; pname_t2 := link d_out fname_base ' expresn.txt' ; f_1 := open pname_f1 "w ; f_2 := open pname_t2 "w ; WHILE (~= ??eof (str := readfile finn)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=#<>$%' EACHLEFT in str)) THEN %write uniStr ; chr1 := str@0 ; IF (= chr_tab chr1) THEN chr1 := ' ' ; ELSE chr1 := link chr1 ' ' ; ENDIF ; %write uniStr ; chr1 := str@0 ; IF (= chr_tab chr1) THEN chr1 := ' ' ; ELSE chr1 := link chr1 ' ' ; ENDIF ; writefile f_1 (link chr1 (string i_line_old)) ; writefile f_2 (link (string i_line_old) ' ' (link uniStr_old) ' ') ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; %writefile fout (link chr1 ' ' (string i_line_old) ' ' uniStr_old ' ') ; EACH close finn f_1 f_2 ; (link (string str@0) ' ' ' ' (EACH string (string i_line_old)) ' ' ' ' uniStr_old ) % ; host 'paste "' pname_t1 '" "' pname_t2 '" >"' pname_out '" ' ; 08********08 #] 02Aug2018 fix unicode mashing # Approach : single -file output sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL finn fout i_line i_line_old str uniStr uniStr_old ; % ; pname_inn := link d_inn fname_base '.txt' ; pname_out := link d_out fname_base ' stdForm.txt' ; finn := open pname_inn "r ; fout := open pname_out "w ; i_line := 0 ; i_line_old := 1 ; uniStr_old := null ; WHILE (~= ??eof (str := readfile finn)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=<>#$%' EACHLEFT in str)) THEN write uniStr ; writefile fout (link (string str@0) ' ' ' ' (EACH string (string i_line_old)) ' ' ' ' uniStr_old) ; %writefile fout uniStr_old ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := sym_stdForm link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; writefile fout (link link 'eof ' (string i_line_old) ' ' uniStr_old ' ') ; EACH close finn fout ; } sym_stdForm_file_test IS { IF flag_break THEN BREAK ; ENDIF ; fout := open '/media/bill/ramdisk/unicode f_tst.txt' "w ; i_line_old := 45283 ; % ; uniStr_old1 := 'sin(Aθpc(POIo(t),t))' ; str := ' = 3*f_BARNES(Vonv(particle),Aθpd(POIo(t),t))*|Q(particle)|*Vons(particle)^2/c*sin(Aθpc(POIo(t),t))*cos(Aθpc(POIo(t),t))/Rpcs(POIp)^3*{{[β*sin(Aθpc(POIo(t),t))]^2/{1 - [β*sin(Aθpd(POIo(t),t))]^2} + 1}*Rodh(Vonv_X_Rpcv(POIo(t),t))' ; uniStr := sym_stdForm str ; uniStr_old1 := link uniStr_old1 ' ' uniStr ; writefile fout 'test 1 result : ' ; %writefile fout (link ((string str@0) ' ' ' ' (EACH string (string i_line_old)) ' ' ' ' uniStr_old1)) ; writefile fout (link (string str@0) ' ' ' ' (string i_line_old) ' ' ' ' uniStr_old1) ; % ; uniStr_old2 := 'sin(Aθpc(POIo(t),t))' ; str := ' = 3*f_BARNES(Vonv(particle),Aθpd(POIo(t),t))*|Q(particle)|*Vons(particle)^2/c*sin(Aθpc(POIo(t),t))*cos(Aθpc(POIo(t),t))/Rpcs(POIp)^3*{{[β*sin(Aθpc(POIo(t),t))]^2/{1 - [β*sin(Aθpd(POIo(t),t))]^2} + 1}*Rodh(Vonv_X_Rpcv(POIo(t),t))' ; uniStr := sym_stdForm str ; uniStr_old2 := sym_stdForm link uniStr_old2 ' ' uniStr ; writefile fout 'test 2 result : ' ; writefile fout (link (string str@0) ' ' ' ' (string i_line_old) ' ' ' ' uniStr_old2) ; % ; close fout ; } >> This showed immediate problems +-----+ 02Aug2018 10:29 current version : sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL finn fout i_line i_line_old str uniStr uniStr_old ; % ; pname_inn := link d_inn fname_base '.txt' ; pname_out := link d_out fname_base ' stdForm.txt' ; finn := open pname_inn "r ; fout := open pname_out "w ; i_line := 0 ; i_line_old := 1 ; uniStr_old := null ; WHILE (~= ??eof (str := readfile finn)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=<>#$%' EACHLEFT in str)) THEN write uniStr ; writefile fout (link (string str@0) ' ' (string i_line_old) ' ' uniStr_old ) ; %writefile fout uniStr_old ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; writefile fout (link (string uniStr@0) ' ' (string i_line_old) ' ' uniStr_old ' ') ; EACH close finn fout ; } >> mashed unicode in "symParen_file_bads test stdForm.txt" : 1 %7)∂[∂(t): Rpcs(POIo(t),t)] % 2 = [ - Rocs(POIo)*cos(AΞoc(POIo))*Vons(particle) + Vons(particle)^2*t] /[Rocs(POIo)^2 - >> however, proper unicode in "symParen_file_bads test stdForm HFLN.txt" >> Why?!!???? This is crazy, as in : symParen_file_bads IS OP d_inn d_out fname_base { LOCAL f_stdFbase f_Lucas f_HFLN ; f_stdFbase := link fname_base ' stdForm' ; f_Lucas := link f_stdFbase ' Lucas' ; f_HFLN := link f_stdFbase ' HFLN' ; sym_stdForm_file d_inn d_out fname_base ; symExtract_file_Lucas d_out d_out f_stdFbase ; symList_good_badParens d_out d_out f_HFLN ; } >> "symList_good_badParens" produces "unmashed unicode" from "mashed unicode" output from "sym_stdForm_file" !!! : +-----+ sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL f_inn f_out i_line i_line_old str uniStr uniStr_old ; % ; IF flag_break THEN BREAK ; ENDIF ; pname_inn := link d_inn fname_base '.txt' ; pname_tmp := link d_out fname_base ' temp.txt' ; pname_out := link d_out fname_base ' stdForm.txt' ; % ; f_inn := open pname_inn "r ; f_out := open pname_tmp "w ; i_line := 0 ; i_line_old := 1 ; uniStr_old := null ; WHILE (~= ??eof (str := readfile f_inn)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=<>#$%' EACHLEFT in str)) THEN write uniStr ; writefile f_out (link (string str@0) ' ' (string i_line_old) ' ' uniStr_old ) ; %writefile f_out uniStr_old ; uniStr_old := uniStr ; i_line_old := i_line ; ELSE uniStr_old := link uniStr_old ' ' uniStr ; ENDIF ; ENDWHILE ; writefile f_out (link (string uniStr@0) ' ' (string i_line_old) ' ' uniStr_old ' ') ; EACH close f_inn f_out ; % ; IF flag_break THEN BREAK ; ENDIF ; f_inn := open pname_tmp "r ; f_out := open pname_out "w ; WHILE (~= ??eof (str := readfile f_inn)) DO writefile f_out str ; %writefile f_out (link str) ; ENDWHILE ; EACH close f_inn f_out ; } +-----+ >> What happens if I just read-write "mashed unicode" output from "sym_stdForm_file"? >> first of wo approaches with f_tmp file with second WHILE read-write : writefile f_out str ; >> gives "mashed unicode" for both "fname_base [ temp.txt,stdForm.txt]" >> BUT "symParen_file_bads test stdForm HFLN badParens.txt" has ""mashed unicode" ??! Now try second of two approaches: writefile f_out (link str) ; >> Same problems as for previous >> simply read-write won't work >> ? why does "unmashed unicode" arise in "symParen_file_bads test stdForm HFLN.txt" from "mashed unicode" in "fname_base stdForm.txt" 03Aug2018 didn't work - need first char for symExtract : sym_stdForm_file IS OP d_inn d_out fname_base { LOCAL f_inn f_out i_line i_line_old str str_out uniStr uniStr_old ; % ; pname_inn := link d_inn fname_base '.txt' ; %pname_tmp := link d_out fname_base ' temp.txt' ; pname_out := link d_out fname_base ' stdForm.txt' ; % ; IF file_exists '-f' pname_inn THEN f_inn := open pname_inn "r ; %f_out := open pname_tmp "w ; f_out := open pname_out "w ; symTrans_lineType := `# ; % at file start, assume comments ; i_line := 0 ; i_line_old := 1 ; uniStr_old := null ; WHILE (~= ??eof (str := readfile f_inn)) DO i_line := i_line + 1 ; uniStr := sym_stdForm str ; IF (OR ('=<>#$%' EACHLEFT in str)) THEN IF (in str@0 '#$%') THEN symTrans_lineType := str@0 ; ENDIF ; %write uniStr ; IF (~= null uniStr_old) THEN str_out := sym_stdForm (link 'l(' (string i_line_old) ') ' uniStr_old ) ; writefile f_out str_out ; ENDIF ; i_line_old := i_line ; IF (= `# symTrans_lineType) THEN uniStr_old := null ; ELSE uniStr_old := uniStr ; ENDIF ; ELSE IF (= `# symTrans_lineType) THEN uniStr_old := null ; ELSE uniStr_old := link uniStr_old ' ' uniStr ; ENDIF ; ENDIF ; ENDWHILE ; IF (AND (~= `# symTrans_lineType) (~= null uniStr)) THEN str_out := sym_stdForm (link 'l(' (string i_line_old) ') ' uniStr_old ' ') ; writefile f_out str_out ; ENDIF ; % ; EACH close f_inn f_out ; write 'sym_stdForm_file - processing executed normally on file :' ; write link ' "' pname_inn '" ' ; l ELSE write 'sym_stdForm_file ERROR - input file is missing :' ; write link ' "' pname_inn '" ' ; o ENDIF } sym_stdForm IS OP str { LOCAL chrList i in_blanks len symNew ; NONLOCAL chrs_blank ; % ; chrList := str_to_unicodeList str ; len := gage shape chrList ; in_blanks := o ; i := 0 ; % ; symNew := null ; WHILE (i < len) DO IF (in chrList@i chrs_blank) THEN in_blanks := l ; IF (NOT in_blanks) THEN symNew := link symNew ' ' ; in_blanks := o ; ENDIF ; ELSEIF (in chrList@i chrs_math_spaced) THEN symNew := link symNew ' ' chrList@i ' ' ; ELSEIF (in chrList@i (solitary ':') ) THEN symNew := link symNew chrList@i ' ' ; ELSE symNew := link symNew chrList@i ; ENDIF ; i := i + 1 ; ENDWHILE ; % ; symNew } 03Aug2018 18:26 OK - looks good!! EXCEPT - extra spaces added to [-,=,+], for example : ∫{∂(Aθpc),0.to.Aθpcf: cos(Aθpc(POIo(t),t))*Rpcs(POIo(t),t)^( - 4)} ∂[∂(Aθpc): Rpcs(POIo(t),t = 0)] ∂[∂(Aθpc): Rpcs(POIo(t),t)^( - β + 1)] >> take them out! This is caused by sym_stdForm 03Aug2018 18:33 revamped sym_stdForm IS OP str { LOCAL blanks chrList i in_blanks len symNew ; NONLOCAL chrs_blank ; % ; blanks := link chrs_blank ; chrList := str_to_unicodeList str ; len := gage shape chrList ; in_blanks := o ; i := 0 ; % ; symNew := null ; WHILE (i < len) DO IF (in chrList@i blanks) THEN in_blanks := l ; IF (NOT in_blanks) THEN symNew := link symNew ' ' ; ENDIF ; ELSEIF (in chrList@i chrs_math_spaced) THEN symNew := link symNew chrList@i ; in_blanks := o ; ELSEIF (in chrList@i (solitary ':') ) THEN symNew := link symNew chrList@i ; in_blanks := o ; ELSE symNew := link symNew chrList@i ; in_blanks := o ; ENDIF ; i := i + 1 ; ENDWHILE ; % ; symNew } Much better - but I still get "chr_space chr_tab" sequences which aren't standard form remove "blanks" : sym_stdForm IS OP str { LOCAL blanks chrList i in_blanks len symNew ; NONLOCAL chrs_blank ; % ; %blanks := link chrs_blank ; chrList := str_to_unicodeList str ; len := gage shape chrList ; in_blanks := o ; i := 0 ; % ; symNew := null ; WHILE (i < len) DO IF (in chrList@i chrs_blank) THEN in_blanks := l ; IF (NOT in_blanks) THEN symNew := link symNew ' ' ; ENDIF ; ELSEIF (in chrList@i chrs_math_spaced) THEN symNew := link symNew chrList@i ; in_blanks := o ; ELSEIF (in chrList@i (solitary ':') ) THEN symNew := link symNew chrList@i ; in_blanks := o ; ELSE symNew := link symNew chrList@i ; in_blanks := o ; ENDIF ; i := i + 1 ; ENDWHILE ; % ; symNew } >> WRONG!!! now there are no spaces! Now it's looking good! sym_stdForm IS OP str { LOCAL blanks chrList i in_blanks len symNew ; NONLOCAL chrs_blank ; % ; blanks := link chrs_blank ; chrListLast := null ; chrList := str_to_unicodeList str ; len := gage shape chrList ; in_blanks := o ; in_HFLN := o ; i := 0 ; % can remove "in_blanks := o" ; symNew := null ; WHILE (i < len) DO IF (in chrList@i chrs_blank) THEN in_blanks := l ; ELSEIF (in chrList@i chrs_symbols_HFLN) THEN IF (OR (in chrListLast chrs_blank) (in chrListLast chrs_paren_stop)) THEN symNew := link symNew ' ' chrList@i ; ELSE symNew := link symNew chrList@i ; ENDIF ; in_HFLN := l ; in_blanks := o ; ELSEIF (in chrList@i chrs_paren_strt) THEN IF (OR (in chrListLast chrs_blank) (in chrListLast chrs_paren_stop)) THEN symNew := link symNew ' ' chrList@i ; ELSE symNew := link symNew chrList@i ; ENDIF ; in_HFLN := o ; in_blanks := o ; ELSEIF (in chrList@i chrs_math_spaced) THEN IF (in (last symNew) blanks) THEN spc_pre := null; ELSE spc_pre := ' ' ; ENDIF ; symNew := link symNew spc_pre chrList@i ' ' ; in_HFLN := o ; in_blanks := o ; ELSEIF (= chrList@i ':') THEN symNew := link symNew chrList@i ' ' ; in_HFLN := o ; in_blanks := o ; ELSE symNew := link symNew chrList@i ; in_HFLN := o ; in_blanks := o ; ENDIF ; chrListLast := chrList@i ; i := i + 1 ; ENDWHILE ; % ; symNew } BUT - I still have extra spaces between parens etc : ([ ...843... = { [Rpcs(POIo(t),t)*cos(Aθpc(POIo(t),t)]^2 + [Rpcs(POIo(t),t)*sin(Aθpc(POIo(t),t))^2}^(1/2) Look at : IF (OR (in chrListLast chrs_blank) (in chrListLast chrs_paren_stop)) THEN BUT, the following "creates" new symbols! : IF (in chrListLast chrs_paren_stop) THEN Go back to the previous, and put in a case for sequential [open,close] parens IF (OR ((AND in_HFLN (in chrListLast chrs_blank)) (in chrListLast chrs_paren_stop)) THEN This "keeps spaced HFLN chars" separated if that was in the code Still problem with some paren-spaces Leave it for now. Time to fix up symbols 08********08 #] 13Nov2018 Restart work after 2 months on IJCNN2018 stuff qnial> lq_symTests >>> loading start : symbols system loaddefs.ndf ?undefined identifier: FINN_EXTENSION := FILEPATH_PATHBASEEXTN <***> PNAME_INN ; PNAME_OUT <<< loading ended : symbols system loaddefs.ndf errors found: 1 >>> loading start : strings.ndf <<< loading ended : strings.ndf >>> loading start : symbols extract.ndf ?undefined identifier: FINN_EXTENSION := FILEPATH_PATHBASEEXTN <***> PNAME_STDFORM_INN ; <<< loading ended : symbols extract.ndf errors found: 1 >>> loading start : symbols translate.ndf ?undefined identifier: FINN_EXTENSION := FILEPATH_PATHBASEEXTN <***> PNAME_STDFORM_INN ; ?undefined identifier: FINN_EXTENSION := FILEPATH_PATHBASEEXTN <***> PNAME_INN ; PNAME_NUM <<< loading ended : symbols translate.ndf errors found: 2 >>> loading start : symbols fix.ndf ?undefined identifier: EXTENSION := FILEPATH_PATHBASEEXTN <***> PNAME_INN ; PNAME_OUT ?undefined identifier: EXTENSION := FILEPATH_PATHBASEEXTN <***> PNAME_INN ; FLAG_SYMDEBUG ?undefined identifier: SYMBAD IS SYMPAREN_FILE_BADS <***> D_LUCA D_QNIAL_TEMP <<< loading ended : symbols fix.ndf errors found: 3 >>> loading start : strings- tests.ndf <<< loading ended : strings- tests.ndf >>> loading start : symbols extract- tests.ndf <<< loading ended : symbols extract- tests.ndf >>> loading start : symbols translate- tests.ndf <<< loading ended : symbols translate- tests.ndf >>> loading start : symbols fix.ndf In ALL d_QNial_mine Change : filepath_pathBaseExtn To : pName_extract_dirBaseExtn check where occurs : $ find "/media/bill/PROJECTS/Qnial/" -maxdepth 2 -name "*.ndf" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number "filepath_pathBaseExtn" "FILE" /media/bill/PROJECTS/Qnial/MY_NDFS/symbols system loaddefs.ndf:142: dinn finn_base finn_extension := filepath_pathBaseExtn pname_inn ; /media/bill/PROJECTS/Qnial/MY_NDFS/symbols fix.ndf:353: path base extension := filepath_pathBaseExtn pname_inn ; /media/bill/PROJECTS/Qnial/MY_NDFS/symbols fix.ndf:453: path base extension := filepath_pathBaseExtn pname_inn ; /media/bill/PROJECTS/Qnial/MY_NDFS/symbols extract.ndf:569: dinn finn_base finn_extension := filepath_pathBaseExtn pname_stdForm_inn ; /media/bill/PROJECTS/Qnial/MY_NDFS/symbols translate.ndf:395: dinn finn_base finn_extension := filepath_pathBaseExtn pname_stdForm_inn ; /media/bill/PROJECTS/Qnial/MY_NDFS/symbols translate.ndf:456: dinn finn_base finn_extension := filepath_pathBaseExtn pname_inn ; >> I replaced ALL $ find "/media/bill/PROJECTS/Qnial/" -maxdepth 2 -name "*.ndf" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number "pName_extract_dirBaseExtn" "FILE" /media/bill/PROJECTS/Qnial/MY_NDFS/conference guides - update inserts.ndf:63:# pName_extract_dirBaseExtn IS OP pName - extract [path, base, extension] from a path-file name /media/bill/PROJECTS/Qnial/MY_NDFS/conference guides - update inserts.ndf:151: dir base extension := pName_extract_dirBaseExtn webPage ; /media/bill/PROJECTS/Qnial/MY_NDFS/file_ops.ndf:27:# pName_extract_dirBaseExtn IS OP pName - extract [path, base, extension] from a path-file name /media/bill/PROJECTS/Qnial/MY_NDFS/file_ops.ndf:30:pName_extract_dirBaseExtn IS OP pName /media/bill/PROJECTS/Qnial/MY_NDFS/file_ops.ndf:39:# pName_extract_dirBaseExtn '/media/bill/PROJECTS/Qnial/MY_NDFS/file_ops.ndf' /media/bill/PROJECTS/Qnial/MY_NDFS/file_ops.ndf:40:# pName_extract_dirBaseExtn 'file_ops.ndf' /media/bill/PROJECTS/Qnial/MY_NDFS/file_ops.ndf:41:# pName_extract_dirBaseExtn '.ndf' /media/bill/PROJECTS/Qnial/MY_NDFS/file_ops.ndf:42:# pName_extract_dirBaseExtn >> keep all as is Try loading again : qnial> lq_symTests >>> loading start : symbols system loaddefs.ndf <<< loading ended : symbols system loaddefs.ndf >>> loading start : strings.ndf <<< loading ended : strings.ndf >>> loading start : symbols extract.ndf <<< loading ended : symbols extract.ndf >>> loading start : symbols translate.ndf <<< loading ended : symbols translate.ndf >>> loading start : symbols fix.ndf <<< loading ended : symbols fix.ndf >>> loading start : strings- tests.ndf <<< loading ended : strings- tests.ndf >>> loading start : symbols extract- tests.ndf <<< loading ended : symbols extract- tests.ndf >>> loading start : symbols translate- tests.ndf <<< loading ended : symbols translate- tests.ndf >>> loading start : symbols fix- tests.ndf <<< loading ended : symbols fix- tests.ndf >> Now everything loads 08********08 #] 22Nov2018 sym_stdForm Change : +-----+ sym_stdForm IS OP str { LOCAL blanks chrList chrListLast i in_blanks len spc strStdForm ; NONLOCAL chrs_blank trnType trnType_old ; % ; chrListLast := null ; chrList := str_to_unicodeList str ; len := gage shape chrList ; i := 0 ; % ; strStdForm := null ; WHILE (i < len) DO IF (= chrList@i '/') THEN IF ((i + 1) < len) THEN IF (= chrList@(i + 1) '*') THEN trnType_old := trnType ; trnType := '/*' ; strStdForm := link strStdForm '/*' ; i := i + 1 ; ELSEIF (= chrList@(i + 1) '$') THEN trnType_old := trnType ; trnType := '/$' ; strStdForm := link strStdForm '/$' ; i := i + 1 ; ELSEIF (= chrList@(i + 1) '%') THEN trnType_old := trnType ; trnType := '/%' ; strStdForm := link strStdForm '/%' ; ELSEIF (= chrList@(i + 1) '^') THEN trnType := trnType_old ; strStdForm := link strStdForm '/%' ; trnType_old := trnType ; i := i + 1 ; ELSE strStdForm := link strStdForm '/' ; ENDIF ; ELSE strStdForm := link strStdForm '/' ; ENDIF ; ELSEIF (in chrList@i chrs_blank) THEN null ; ELSEIF (in chrList@i chrs_symbols_HFLN) THEN IF (OR (in chrListLast chrs_blank) (in chrListLast chrs_paren_stop)) THEN strStdForm := link strStdForm ' ' chrList@i ; ELSE strStdForm := link strStdForm chrList@i ; ENDIF ; ELSEIF (in chrList@i chrs_math_spaced) THEN IF (in chrListLast chrs_blank) THEN spc := null ; ELSE spc := ' ' ; ENDIF ; strStdForm := link strStdForm spc chrList@i ' ' ; ELSEIF (= chrList@i ':') THEN strStdForm := link strStdForm chrList@i ' ' ; ELSE strStdForm := link strStdForm chrList@i ; ENDIF ; chrListLast := last strStdForm ; i := i + 1 ; ENDWHILE ; % ; strStdForm } +-----+ To old version : +-----+ # sym_stdForm IS OP str - to put [symbols,expressions] in a standard format for comparison - mostly remove [spaces,tabs] # 24Apr2018 initial # 24May2018 modified to handle unicode, code taken from symExtract_string # 26Jun2018 revamped to simply convert str to chrList, get rid of chrStr_get # 03Sep2018 NOTE : sym_stdForm RETAINS comments intact # remove all [spaces,tabs] EXCEPT ensure (force) that : # 1 space around each chrs_math_spaced= EACH string '+-=' # 1 space after '.:' and chrs_paren_stop= EACH string ')]}>' # 1 space between symAtomics = sequences of chrs_symbols_HFLN= # You can legitimately have "symAtomic space (" for example : # between symAtomic and subsequent chars_paren_strt= EACH string '([{<' sym_stdForm IS OP str { LOCAL blanks uniList i in_blanks len symNew ; NONLOCAL chrs_blank ; % ; blanks := link chrs_blank ; uniListLast := null ; uniList := str_to_unicodeList str ; len := gage shape uniList ; in_blanks := o ; in_HFLN := o ; i := 0 ; % can remove "in_blanks := o" ; symNew := null ; WHILE (i < len) DO IF (in uniList@i chrs_blank) THEN in_blanks := l ; ELSEIF (in uniList@i chrs_symbols_HFLN) THEN IF (OR (in uniListLast chrs_blank) (in uniListLast chrs_paren_stop)) THEN symNew := link symNew ' ' uniList@i ; ELSE symNew := link symNew uniList@i ; ENDIF ; in_HFLN := l ; in_blanks := o ; ELSEIF (in uniList@i chrs_paren_strt) THEN IF (OR (in uniListLast chrs_blank) (in uniListLast chrs_paren_stop)) THEN symNew := link symNew ' ' uniList@i ; ELSE symNew := link symNew uniList@i ; ENDIF ; in_HFLN := o ; in_blanks := o ; ELSEIF (in uniList@i chrs_paren_stop) THEN IF (OR (in uniListLast chrs_blank) (in uniListLast chrs_paren_stop)) THEN symNew := link symNew ' ' uniList@i ; ELSE symNew := link symNew uniList@i ; ENDIF ; in_HFLN := o ; in_blanks := o ; ELSEIF (in uniList@i chrs_math_spaced) THEN IF (in (last symNew) blanks) THEN spc_pre := null; ELSE spc_pre := ' ' ; ENDIF ; symNew := link symNew spc_pre uniList@i ' ' ; in_HFLN := o ; in_blanks := o ; ELSEIF (= uniList@i ':') THEN symNew := link symNew uniList@i ' ' ; in_HFLN := o ; in_blanks := o ; ELSE symNew := link symNew uniList@i ; in_HFLN := o ; in_blanks := o ; ENDIF ; uniListLast := uniList@i ; i := i + 1 ; ENDWHILE ; % ; symNew } +-----+ 08********08 #] 22Nov2018 strings_alltest results and fixes - reset code back to early September when it worked Summary of test results : /media/bill/PROJECTS/Qnial/code develop_test/181122 14h20m00s alltest strings.txt, date= 181122 14h20m # str_to_unicodeList_test example [1,2,3,4,5,6] : OK - result matches standard # string_sub_test example [1,2,2a,2b,2c,3,3a,3b,3c,4,4a,4b,4c : OK - result matches standard # sym_stdForm_test example 28 : OK - result matches standard # ALL other examples FAILED [1-9,20-27] >> Problems : - initial spaces are retained - doubling of spaces after chrs_math_spaced 26Nov2018 remove old code : +-----+ # old code ELSEIF (in chrList@i chrs_paren_stop) THEN IF (OR (in chrListLast chrs_blank) (in chrListLast chrs_paren_stop)) THEN str := link str ' ' chrList@i ; ELSE str := link str chrList@i ; ENDIF ; IF (in chrList@i chrs_blank) THEN s_out := link s_out spc ; spc := '' ; ELSEIF (in chrList@i chrs_paren_strt) THEN s_out := link s_out spc chrList@i ; spc := '' ; ELSEIF (in chrList@i chrs_paren_strt) THEN IF (OR (in chrListLast chrs_blank) (in chrListLast chrs_paren_stop)) THEN s_out := link s_out ' ' chrList@i ; spc := '' ; ELSE s_out := link s_out spc chrList@i ; spc := '' ; ENDIF ; ELSEIF (= ' ' hf) ??????????? THEN s_out := link s_out spc chrList@i ; spc := '' ; hf := l ; IF (OR (in chrListLast chrs_blank) (in chrListLast chrs_paren_stop)) THEN s_out := link s_out ' ' chrList@i ; spc := '' ; hf := l ; ELSEIF (in chrList@i chrs_symbols_HFLN) THEN IF (in chrListLast chrs_blank) THEN s_out := link s_out ' ' chrList@i ; spc := '' ; hf := l ; ELSE s_out := link s_out spc chrList@i ; spc := '' ; hf := l ; ENDIF ; +-----+ # full test list # enddoc