Skip to content

Commit cf0c434

Browse files
GhidorahRexGhidorahRex
committed
Merge remote-tracking branch 'origin/GT-3414_dev747368_slow_stringtable'
Fixes NationalSecurityAgency#1259
2 parents 4d29a38 + 66fc367 commit cf0c434

File tree

8 files changed

+369
-63
lines changed

8 files changed

+369
-63
lines changed

‎Ghidra/Features/Base/ghidra_scripts/TranslateStringsScript.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import ghidra.program.model.data.TranslationSettingsDefinition;
2222
import ghidra.program.model.listing.Data;
2323
import ghidra.program.util.DefinedDataIterator;
24+
import util.CollectionUtils;
2425

2526
public class TranslateStringsScript extends GhidraScript {
2627

@@ -39,7 +40,8 @@ public void run() throws Exception {
3940
int count = 0;
4041
monitor.initialize(currentProgram.getListing().getNumDefinedData());
4142
monitor.setMessage("Translating strings");
42-
for (Data data : DefinedDataIterator.definedStrings(currentProgram, currentSelection)) {
43+
for (Data data : CollectionUtils.asIterable(
44+
DefinedDataIterator.definedStrings(currentProgram, currentSelection))) {
4345
if (monitor.isCancelled()) {
4446
break;
4547
}

‎Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/strings/ViewStringsTableModel.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import ghidra.program.model.listing.*;
3131
import ghidra.program.util.*;
3232
import ghidra.util.StringUtilities;
33+
import ghidra.util.Swing;
3334
import ghidra.util.datastruct.Accumulator;
3435
import ghidra.util.exception.CancelledException;
3536
import ghidra.util.table.AddressBasedTableModel;
@@ -111,8 +112,8 @@ protected void doLoad(Accumulator<ProgramLocation> accumulator, TaskMonitor moni
111112
Listing listing = localProgram.getListing();
112113

113114
monitor.setCancelEnabled(true);
114-
monitor.initialize((int) listing.getNumDefinedData());
115-
115+
monitor.initialize(listing.getNumDefinedData());
116+
Swing.allowSwingToProcessEvents();
116117
for (Data stringInstance : DefinedDataIterator.definedStrings(localProgram)) {
117118
accumulator.add(createIndexedStringInstanceLocation(localProgram, stringInstance));
118119
monitor.checkCanceled();

‎Ghidra/Features/Base/src/main/java/ghidra/program/database/ProgramBuilder.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -695,6 +695,14 @@ public Namespace createClassNamespace(String name, String parentNamespace, Sourc
695695
return c;
696696
}
697697

698+
public void applyFixedLengthDataType(String addressString, DataType dt, int length)
699+
throws CodeUnitInsertionException {
700+
startTransaction();
701+
DataUtilities.createData(program, addr(addressString), dt, length, false,
702+
ClearDataMode.CLEAR_ALL_CONFLICT_DATA);
703+
endTransaction();
704+
}
705+
698706
public void applyDataType(String addressString, DataType dt) {
699707
applyDataType(addressString, dt, 1);
700708
}
@@ -874,7 +882,7 @@ else if (encoding == StandardCharsets.UTF_16BE || encoding == StandardCharsets.U
874882
}
875883

876884
public Data createString(String address, String string, Charset charset, boolean nullTerminate,
877-
AbstractStringDataType dataType) throws Exception {
885+
DataType dataType) throws Exception {
878886
if (nullTerminate) {
879887
string = string + "\0";
880888
}
@@ -883,7 +891,7 @@ public Data createString(String address, String string, Charset charset, boolean
883891
}
884892

885893
public Data createString(String address, byte[] stringBytes, Charset charset,
886-
AbstractStringDataType dataType) throws Exception {
894+
DataType dataType) throws Exception {
887895
Address addr = addr(address);
888896
setBytes(address, stringBytes);
889897
if (dataType != null) {

‎Ghidra/Features/Base/src/test.slow/java/ghidra/app/plugin/core/string/DefinedStringIteratorTest.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,17 @@ public void setUp() throws Exception {
7272
builder.createEncodedString("0x500", "This is the last string", StandardCharsets.US_ASCII,
7373
false);
7474

75+
ArrayDataType charArray = new ArrayDataType(new CharDataType(), 50, 1);
76+
builder.createString("0x600", "The 600 chararray", StandardCharsets.US_ASCII, true,
77+
charArray);
78+
7579
// create an empty area for tests to do their own thing
76-
builder.createUninitializedMemory("uninitialized", "0x3000", 100);
80+
builder.createUninitializedMemory("uninitialized", "0x3000", 0x1000);
81+
builder.applyDataType("0x3100", charArray);
82+
builder.applyFixedLengthDataType("0x3200", new StringDataType(), 10);
7783

7884
program = builder.getProgram();
85+
7986
}
8087

8188
@Test
@@ -117,6 +124,11 @@ public void testIterator() throws Exception {
117124
assertEquals(addr(0x500), foundString.getAddress());
118125
assertEquals("This is the last string", foundString.getString(program.getMemory()));
119126

127+
assertTrue(iterator.hasNext());
128+
foundString = iterator.next();
129+
assertEquals(addr(0x600), foundString.getAddress());
130+
assertEquals("The 600 chararray", foundString.getString(program.getMemory()));
131+
120132
assertFalse(iterator.hasNext());
121133
}
122134

‎Ghidra/Features/Base/src/test/java/ghidra/program/util/DefinedDataIteratorTest.java

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/* ###
2+
* IP: GHIDRA
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package ghidra.program.util;
17+
18+
import static org.junit.Assert.*;
19+
20+
import java.nio.charset.StandardCharsets;
21+
import java.util.List;
22+
23+
import org.junit.Before;
24+
import org.junit.Test;
25+
26+
import ghidra.program.database.ProgramDB;
27+
import ghidra.program.model.data.*;
28+
import ghidra.program.model.listing.Data;
29+
import ghidra.program.model.util.CodeUnitInsertionException;
30+
import ghidra.test.AbstractGhidraHeadlessIntegrationTest;
31+
import ghidra.test.ToyProgramBuilder;
32+
import util.CollectionUtils;
33+
34+
public class DefinedDataIteratorTest extends AbstractGhidraHeadlessIntegrationTest {
35+
36+
private ToyProgramBuilder builder;
37+
private ProgramDB program;
38+
private DataTypeManager dtm;
39+
private DataType intDT;
40+
private StringDataType stringDT;
41+
private CharDataType charDT;
42+
private DataType charArray;
43+
private StructureDataType struct1DT;
44+
private ArrayDataType structArray;
45+
private StructureDataType struct2DT;
46+
private TypeDef intTD;
47+
48+
@Before
49+
public void setUp() throws Exception {
50+
51+
builder = new ToyProgramBuilder("DefinedDataIteratorTests", false);
52+
program = builder.getProgram();
53+
dtm = program.getDataTypeManager();
54+
55+
intDT = AbstractIntegerDataType.getSignedDataType(4, dtm);
56+
intTD = new TypedefDataType("int_typedef", intDT);
57+
stringDT = StringDataType.dataType;
58+
charDT = new CharDataType(dtm);
59+
charArray = new ArrayDataType(charDT, 20, charDT.getLength());
60+
61+
struct1DT = new StructureDataType("struct1", 100);
62+
struct1DT.replaceAtOffset(0, intDT, intDT.getLength(), "f1", null);
63+
struct1DT.replaceAtOffset(10, charArray, charArray.getLength(), "f2", null);
64+
struct1DT.replaceAtOffset(50, stringDT, 10, "f3", null);
65+
66+
structArray = new ArrayDataType(struct1DT, 10, struct1DT.getLength());
67+
68+
struct2DT = new StructureDataType("struct2", 200);
69+
struct2DT.replaceAtOffset(0, intDT, intDT.getLength(), "f1", null);
70+
struct2DT.replaceAtOffset(10, struct1DT, intDT.getLength(), "f2", null);
71+
72+
builder.createMemory("test", "0x0", 0x2000);
73+
program = builder.getProgram();
74+
}
75+
76+
@Test
77+
public void test_Ints() throws Exception {
78+
builder.applyFixedLengthDataType("0x0", intDT, intDT.getLength());
79+
builder.createString("0x10", "test1", StandardCharsets.UTF_8, true, stringDT);
80+
builder.applyFixedLengthDataType("0x100", struct1DT, struct1DT.getLength());
81+
82+
List<Data> list = CollectionUtils.asList((Iterable<Data>)
83+
DefinedDataIterator.byDataType(program, dt -> dt instanceof IntegerDataType));
84+
85+
assertTrue(list.get(0).getAddress().getOffset() == 0x0);
86+
assertTrue(list.get(1).getAddress().getOffset() == 0x100);
87+
88+
assertEquals(2, list.size());
89+
}
90+
91+
@Test
92+
public void test_Strings() throws Exception {
93+
builder.applyFixedLengthDataType("0x0", intDT, intDT.getLength());
94+
builder.createString("0x10", "test1", StandardCharsets.UTF_8, true, stringDT);
95+
builder.applyFixedLengthDataType("0x100", struct1DT, struct1DT.getLength());
96+
97+
List<Data> list =
98+
CollectionUtils.asList((Iterable<Data>) DefinedDataIterator.definedStrings(program));
99+
100+
assertTrue(list.get(0).getAddress().getOffset() == 0x10);
101+
assertTrue(list.get(1).getAddress().getOffset() == 0x100 + 10);
102+
assertTrue(list.get(2).getAddress().getOffset() == 0x100 + 50);
103+
104+
assertEquals(3, list.size());
105+
}
106+
107+
@Test
108+
public void test_ArrayOfStructs() throws Exception {
109+
builder.applyFixedLengthDataType("0x0", intDT, intDT.getLength());
110+
builder.createString("0x10", "test1", StandardCharsets.UTF_8, true, stringDT);
111+
builder.applyFixedLengthDataType("0x100", structArray, structArray.getLength());
112+
113+
int numElements = structArray.getNumElements();
114+
int lastEle = numElements - 1;
115+
int elementSize = structArray.getElementLength();
116+
117+
List<Data> list =
118+
CollectionUtils.asList((Iterable<Data>) DefinedDataIterator.definedStrings(program));
119+
120+
assertEquals(list.get(0).getAddress().getOffset(), 0x10);
121+
assertEquals(list.get(1 + 0).getAddress().getOffset(), 0x100 + 10);
122+
assertEquals(list.get(1 + 1).getAddress().getOffset(), 0x100 + 50);
123+
124+
assertEquals(list.get(1 + (lastEle * 2) + 0).getAddress().getOffset(),
125+
0x100 + (elementSize * lastEle) + 10);
126+
assertEquals(list.get(1 + (lastEle * 2) + 1).getAddress().getOffset(),
127+
0x100 + (elementSize * lastEle) + 50);
128+
129+
assertEquals(1 + (numElements * 2), list.size());
130+
}
131+
132+
@Test
133+
public void test_Typedefs() throws CodeUnitInsertionException {
134+
// 3 ints: 2 are typedefs, 1 is regular int
135+
builder.applyFixedLengthDataType("0x0", intTD, intTD.getLength());
136+
builder.applyFixedLengthDataType("0x10", intTD, intTD.getLength());
137+
builder.applyFixedLengthDataType("0x20", intDT, intTD.getLength());
138+
139+
// iterating by data type ignores typedefs, so we should get all 3 ints
140+
List<Data> list = CollectionUtils.asList((Iterable<Data>)
141+
DefinedDataIterator.byDataType(program, dt -> dt instanceof IntegerDataType));
142+
143+
assertEquals(3, list.size());
144+
145+
// iterating by data instance, we can inspect the actual data type and get the
146+
// typedef
147+
list = CollectionUtils.asList((Iterable<Data>) DefinedDataIterator.byDataInstance(program,
148+
data -> data.getDataType() instanceof TypeDef));
149+
assertEquals(2, list.size());
150+
}
151+
}

‎Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/StringDataInstance.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,25 @@ public static boolean isString(Data data) {
6565
return false;
6666
}
6767

68+
/**
69+
* Returns true if the specified {@link DataType} is (or could be) a
70+
* string.
71+
* <p>
72+
* Arrays of char-like elements (see {@link ArrayStringable}) are treated
73+
* as string data types. The actual data instance needs to be inspected
74+
* to determine if the array is an actual string.
75+
* <p>
76+
* @param dt DataType to test
77+
* @return boolean true if data type is or could be a string
78+
*/
79+
public static boolean isStringDataType(DataType dt) {
80+
if (dt instanceof TypeDef) {
81+
dt = ((TypeDef) dt).getBaseDataType();
82+
}
83+
return dt instanceof AbstractStringDataType || (dt instanceof Array &&
84+
ArrayStringable.getArrayStringable(((Array) dt).getDataType()) != null);
85+
}
86+
6887
/**
6988
* Returns true if the {@link Data} instance is one of the many 'char' data types.
7089
*

‎Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/listing/DataIterator.java

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
*/
1616
package ghidra.program.model.listing;
1717

18+
import java.util.Arrays;
1819
import java.util.Iterator;
19-
import java.util.NoSuchElementException;
2020

2121
import util.CollectionUtils;
2222

@@ -25,8 +25,18 @@
2525
*
2626
* @see CollectionUtils#asIterable
2727
*/
28-
public interface DataIterator extends Iterator<Data>, Iterable<Data> {
29-
public static final DataIterator EMPTY = createEmptyIterator();
28+
public interface DataIterator extends Iterator<Data>, Iterable<Data> {
29+
public static final DataIterator EMPTY = of(/*nothing*/);
30+
31+
/**
32+
* Create a DataIterator that returns a sequence of the specified items.
33+
*
34+
* @param dataInstances variable length list of items that will be iterated
35+
* @return new Iterator
36+
*/
37+
public static DataIterator of(Data... dataInstances) {
38+
return new IteratorWrapper(Arrays.asList(dataInstances).iterator());
39+
}
3040

3141
@Override
3242
public boolean hasNext();
@@ -40,15 +50,25 @@ default Iterator<Data> iterator() {
4050
}
4151

4252
// --------------------------------------------------------------------------------
43-
// Helper static methods
53+
// Helper static stuff
4454
// --------------------------------------------------------------------------------
45-
public static DataIterator createEmptyIterator() {
46-
return new DataIterator() {
47-
//@formatter:off
48-
@Override public Data next() { throw new NoSuchElementException(); }
49-
@Override public void remove() { throw new IllegalStateException(); }
50-
@Override public boolean hasNext() { return false; }
51-
//@formatter:on
52-
};
55+
56+
static class IteratorWrapper implements DataIterator {
57+
private Iterator<Data> it;
58+
59+
IteratorWrapper(Iterator<Data> it) {
60+
this.it = it;
61+
}
62+
63+
@Override
64+
public boolean hasNext() {
65+
return it.hasNext();
66+
}
67+
68+
@Override
69+
public Data next() {
70+
return it.next();
71+
}
72+
5373
}
5474
}

0 commit comments

Comments
 (0)