1unsigned parsetokens1main (void)
2{
3	unsigned i, j, k, l, m;
4	unsigned curstrlen, curtoklen;
5	unsigned newtok;
6	int bestcurtok, bestcurtoklen;
7	//struct tok2buf * besttok[16];
8	char *besttokptr;
9	curstr=strings;
10	while (curstr) {
11		curstrlen=curstr->inlen;
12		//curstr->seg=curseg;
13		for (i=0; curstrlen>=1 && i<curstrlen;) {
14			if (0 && !tok2bufsize) {
15				//addtotok1(curstr->in, 3);
16			} else {
17				bestcurtok=-1; newtok=1; bestcurtoklen=3;m=0;
18				for (j=k=0; j<tok2bufsize; j++) {
19					if (!memcmp(&curstr->in[i], tok2buf[j].token, tok2buf[j].len)) {
20						//bestcurtok=j; bestcurtoklen=tok2buf[j].len;
21						l=getlenmatch(tok2buf[j].token, &curstr->in[i]);
22						if (tok2buf[j].cu==curstr) {
23							// if (tok2buf[j].len+l>i) //continue;
24							// l=tok2buf[j].len-i;
25							//if (tok2buf[j].token+l>&curstr->in[i]) l=&curstr->in[i]-tok2buf[j].token;
26						}
27						//if ((int)(l=getlenmatch(tok2buf[j].token, &curstr->in[i]))>bestcurtoklen) {
28						if (l>=bestcurtoklen && l==tok2buf[j].len) {
29							newtok=0;
30							bestcurtok=j;
31							bestcurtoklen=l;
32						} else if (l>bestcurtoklen) {
33putchar('.');
34							newtok=1; m=1;
35							bestcurtok=j;
36							bestcurtoklen=l;
37						}
38					}
39				} if (newtok) {
40					addtotok1 (&curstr->in[i], bestcurtoklen);
41				} else if (bestcurtok>=0 && bestcurtoklen>=4) {
42					tok2buf[bestcurtok].occur++;
43				}
44			} //i+=bestcurtoklen;
45			if (bestcurtoklen>=4) i+=bestcurtoklen;
46			else i++;
47printf ("<%d>\n", bestcurtoklen);
48		}
49		curstr=curstr->next;
50	}
51printf ("# tokens before sort: %d\n", tok2bufsize);
52	sorttokens1();
53	compresstoks();
54	//compresstoks2();
55	//compresstoksbpe();
56	curstring=strings;
57	while (curstring) {
58		complit_5a();
59		curstring=curstring->next;
60	}		

61	writetokstofile();
62	writestrs();
63	return 0;
64}

1void sorttokens1 (void)
2{
3	unsigned i, j, k, l;
4	struct tok2buf tmpswaptok;
5unsigned char c[64];
6	for (i=0; i<tok2bufsize; i++) {
7		tok2buf[i].saved=((tok2buf[i].len)*(tok2buf[i].occur+1));
8		if (tok2buf[i].occur<6 || tok2buf[i].len<3) tok2buf[i].saved=0;
9	}
10	for (i=0; i<tok2bufsize-1; i++) {
11		k=i;
12		for (j=i+1; j<tok2bufsize; j++) if (tok2buf[j].saved>tok2buf[k].saved) k=j;
13		if (k!=i) {
14			memcpy (&tmpswaptok, &tok2buf[i], sizeof(tmpswaptok));
15			memcpy (&tok2buf[i], &tok2buf[k], sizeof(tmpswaptok));
16			memcpy (&tok2buf[k], &tmpswaptok, sizeof(tmpswaptok));
17		}
18	} 
19	for (i=0; i<tok2bufsize; i++) {
20		//tok2buf[i].saved=((tok2buf[i].len-1)*(tok2buf[i].occur))-(tok2buf[i].len+1);
21		memcpy (c, tok2buf[i].token, tok2buf[i].len);
22		c[tok2buf[i].len]=0;
23		printf (" Token# %d: \"%s\", Occur %d\n", i, c, tok2buf[i].occur);
24	}
25getchar();
26	for (i=0; i<tok2bufsize; i++) {
27		if (tok2buf[i].saved<1) {tok2bufsize=i; break;}
28	} if (tok2bufsize>128) tok2bufsize=128;
29printf ("# tokens after sort: %d\n", tok2bufsize);
30	collecttokens();
31puts ("b");
32}