CSAW 2024 - Archeology

28th Mar 2025
2 min read
Tags:
ctf,
reversing,
virtual machine

0x00. Introduction

➜  ls
chal  hieroglyphs.txt  message.txt
➜  file chal 
chal: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=bbcf6cbfa61b68c7b8bfa3145852d06580a55643, for GNU/Linux 3.2.0, not stripped

0x01. Concept

int __fastcall main(int argc, const char **argv, const char **envp)
{
  ...
  if ( argc == 2 )
  {
    *(_DWORD *)magic = 0xDDCCBBAA;
    magic[4] = 0xEE;
    plain = (char *)argv[1];
    plain_len = strlen(plain);
    index = 0;
    printf("Encrypted data: ");
    washing_machine(plain, plain_len);
    ...
  }
}

First, the binary passes the input argument to washing_machine() when executed.

int __fastcall main(int argc, const char **argv, const char **envp)
{
    ...
    for ( i = 0; i < plain_len; ++i )
    {
      buf[index] = 0; 
      buf[index + 1] = 1;
      buf[index + 2] = plain[i];
      index += 3;
      for ( j = 0; j <= 9; ++j )
      {
        buf[index] = 0; 
        buf[index + 1] = 0;
        buf[index + 2] = magic[(10 * i + j) % 5];
        buf[index + 3] = 8; 
        buf[index + 4] = 1;
        buf[index + 5] = 3;
        buf[index + 6] = 3; 
        buf[index + 7] = 1;
        buf[index + 8] = 1; 
        buf[index + 9] = 1;
        buf[index + 10] = 0;
        buf[index + 11] = 2; 
        buf[index + 12] = 1;
        buf[index + 13] = 3;
        index += 14;
      }
      buf[index] = 4; 
      buf[index + 1] = 1;
      buf[index + 2] = i;
      index += 3;
    }
    buf[index] = 7; 
    runnnn(buf);
    washing_machine((char *)memory, plain_len);
    ...
}

Next, it inserts the plain values modified by washing_machine() into buf byte by byte like a template. This buf is passed as an argument to runnnn(), which performs operations based on the values internally.

In other words, the binary is a virtual machine with a predefined instruction set, and inserting values into buf is essentially creating shellcode.

Goal

int __fastcall main(int argc, const char **argv, const char **envp)
{
    ...
    stream = fopen("hieroglyphs.txt", "r");
    if ( stream )
    {
      for ( k = 0; fgets(&symbols[256 * k], 256, stream) && k <= 255; ++k )
        *(&savedregs + 256 * k + strcspn(&symbols[256 * k], "\n") - (char *)&unk_12730) = 0;
      fclose(stream);
      for ( m = 0; m < plain_len; ++m )
        printf("%s", &symbols[256 * memory[m]]);
      putchar(10);
      exit(0);
    }
    perror("Failed to open hieroglyphs.txt");
    return 1;
    ...
}

After computation, it reads hieroglyphs.txt and for some reason loads hieroglyph Unicode values every 256 bytes. Then it reads memory values and prints the corresponding hieroglyph.

To clarify, if memory[0] contains 123, it prints the 123rd hieroglyph.

The goal is to reverse-engineer what plaintext input produces the content written in message.txt.

0x02. Exploit

The ciphertext generation process in pseudocode:

tmp = washing_machine(plain)
shellcode = generate_shellcode(tmp)
runnnn(shellcode)
cipher = washing_machine(memory)

Setting aside generate_shellcode(), we need to reverse the operations of washing_machine() and runnnn().

washing_machine

unsigned __int64 __fastcall washing_machine(char *data, unsigned __int64 data_len)
{
  ...
  curr = *data;
  for ( i = 1LL; i < data_len; ++i )
  {
    next = curr ^ data[i];
    data[i] = next;
    curr = next;
  }
  for ( j = 0LL; ; ++j )
  {
    result = data_len >> 1;
    if ( j >= data_len >> 1 )
      break;
    tmp = data[j];
    data[j] = data[data_len - j - 1];
    data[data_len - j - 1] = tmp;
  }
  return result;
}

It performs two main operations:

data[i] = data[i] ^ data[i + 1]
String reversal

To decrypt this operation:

def dewashing_machine(data):
    res = data[::-1]
    for i in range(len(res) - 1, 0, -1):
        res[i] = res[i] ^ res[i - 1]
    return res

runnnn

int __fastcall runnnn(__int64 start)
{
  ...
  cnt = 0;
  flag = 1;
  while ( flag )
  {
    index = cnt++;
    opcode = *(index + start);
    switch ( opcode )
    {
      case 0u:                                  // mov regs[op1], op2
        op1 = *(start + cnt);
        op2 = *(start + cnt + 1);
        cnt += 2;
        regs[op1] = op2;
        break;
      ...
      default:
        LODWORD(v2) = puts("Invalid instruction");
        flag = 0;
        break;
    }
  }
}

runnnn() looks like the above, but it’s too long to show everything. Here’s the summary:

opcode	instruction
0	`mov regs[op1], op2`
1	`xor regs[op1], regs[op2]`
2	`rol regs[op1], op2`
3	`sbox regs[op1]`
4	`mov memory[op2], regs[op1]`
8	`mov regs[op1], memory[op2]`
6	`print regs[op1]`
7	`exit`
8	`ror regs[op1], op2`

Note that sbox regs[op1] is an instruction that fetches the value at index regs[op1] from sbox. Interpreting buf based on this:

for ( i = 0; i < plain_len; ++i )
{
  mov regs[1], plain[i];
  for ( j = 0; j < 10; ++j )
  {
    mov regs[0], AA | BB | CC | DD | EE
    ror regs[1], 3
    sbox regs[1]
    xor regs[1], regs[0]
    rol regs[1], 3
  }
  mov memory[i], reg[1]
}
exit