hack.lu 2011 First Pre-CTF challenge

Category: reversing

binary

Summary: hash algorithm analyze, crackme

Main function in program very clear. It asks a password, calculates hash function by it and compares result with 0xFBF3512A

int main(int argc, const char **argv, const char **envp)
{
  bool flag; // eax@1
  int pass[4]; // [sp+0h] [bp-18h]@1
  unsigned int cookie; // [sp+10h] [bp-8h]@1

  cookie = (unsigned int)&v11 ^ __security_cookie;
  sub_411C0();
  printf("Enter your password to continue\n", 0);
  printf("Password: ", pass[0]);
  scanf("%d-%d-%d-%d", &pass[0], &pass[1], &pass[2], &pass[3]);
  flag = check_pass(pass);
  if ( flag == 1 )
    printf("Gratz. Welcome\n", pass[0]);
  else
    printf("Outch. Try again\n", pass[0]);
  sub_417E3();
  sub_417E3();
  return 0;
}

Function check_pass is not so big, but it calls different function:

bool __cdecl check_pass(int *pass)
{
  signed int i; // [sp+0h] [bp-8h]@1
  int a; // [sp+4h] [bp-4h]@1
  int b; // [sp+4h] [bp-4h]@5

  a = 0;
  for ( i = 0; i < 4; ++i )
  {
    if ( i % 2 )
      a += pass[i];
    b = pass[i] * a;
    if ( i % 3 )
      b -= pass[i];
    a = pass[i] ^ b ^ (unsigned __int8)arr[(pass[i] ^ (unsigned int)b) % 0xE6];
  }
  return (a | calc_second_hash(pass)) == 0xFBF3512A;
}
int __cdecl calc_second_hash(int *pass)
{
  char v1; // ST07_1@3
  int v2; // ST14_4@5
  unsigned __int8 *v4; // [sp+Ch] [bp-1Ch]@2
  unsigned int v5; // [sp+10h] [bp-18h]@1
  int v6; // [sp+14h] [bp-14h]@1
  int v7; // [sp+18h] [bp-10h]@1
  int v8; // [sp+1Ch] [bp-Ch]@1
  int v9; // [sp+20h] [bp-8h]@1
  unsigned int v10; // [sp+24h] [bp-4h]@1
  int v11; // [sp+28h] [bp+0h]@1

  v10 = (unsigned int)&v11 ^ __security_cookie;
  v7 = dword_4C990;
  v8 = dword_4C994;
  v9 = dword_4C998;
  v6 = 0x12344321u;
  v5 = 0;
  while ( 1 )
  {
    v4 = (unsigned __int8 *)&v7;
    do
      v1 = *v4++;
    while ( v1 );
    if ( v5 >= v4 - (unsigned __int8 *)((char *)&v7 + 1) )
      break;
    v2 = v6 ^ *((_BYTE *)&v7 + v5++);
    v6 = 16 * v2;
  }
  return (v6 | 0x87654321) & (pass[3] ^ pass[2] ^ pass[1] ^ *pass);
}

At first glance it is just non-packed crackme and we should bruteforce hash or find a weak in hash algorithm.
But if you try to do it you will get stuck. There are several pitfalls in this task.

First one is real-time patch of check_pass function. Program patches call instruction at 0x00041167
When you look in IDA it is

.text:00041163 loc_41163:
.text:00041163             mov     eax, [ebp+pass]
.text:00041166             push    eax
.text:00041167
.text:00041167 loc_41167:
.text:00041167 call sub_41000
.text:0004116C             add     esp, 4
.text:0004116F             or      eax, [ebp+a]
.text:00041172             mov     [ebp+a], eax
.text:00041175             cmp     [ebp+a], 0FBF3512Ah
.text:0004117C             jnz     short loc_41185
.text:0004117E             mov     eax, 1
.text:00041183             jmp     short loc_41187

When you execute program and stop at breakpoint 0x00041167 it is

.text:00041163 loc_41163:
.text:00041163             mov     eax, [ebp+pass]
.text:00041166             push    eax
.text:00041167
.text:00041167 loc_41167:
.text:00041167 call sub_4EEC5
.text:0004116C             add     esp, 4
.text:0004116F             or      eax, [ebp+a]
.text:00041172             mov     [ebp+a], eax
.text:00041175             cmp     [ebp+a], 0FBF3512Ah
.text:0004117C             jnz     short loc_41185
.text:0004117E             mov     eax, 1
.text:00041183             jmp     short loc_41187

So calc_second_hash is different when you execute the program. It looks like

int __cdecl sub_4EEC5(int *pass)
{
  int v1; // edx@9
  int v2; // ST04_4@10
  int v3; // ecx@10
  int result; // eax@11
  unsigned int i; // [sp+0h] [bp-8h]@1
  int hash; // [sp+4h] [bp-4h]@1
  int v7; // [sp+4h] [bp-4h]@3
  int v8; // [sp+4h] [bp-4h]@10

  hash = 0;
  for ( i = 0; i < 4; ++i )
  {
    v7 = pass[i] + hash;
    if ( i % 3 )
      v7 *= pass[i];
    if ( i % 4 )
      v7 += pass[i];
    if ( i % 5 )
      v7 ^= pass[i];
    v1 = v7;
    hash = v1 & 0xFFFFF;
  }
  v2 = hash ^ 0x8F6DD;
  v3 = v2 == 0;
  v8 = v2 == 0;
  if ( v8 == 1 )
    result = v3;
  else
    result = v8;
  return result;
}

It is easier then previous version and also it has a secret =)

If you look at stack at the end of function it would be strange thing there.

It adds 0x1ch to esp and jump to main function if hash is ok!

GREAT! For solve task we just need calculate one easy function instead 2 difficult one!

So one possible answer is

Enter your password to continue
Password: 587478-1-1-1
Gratz. Welcome

 

 

Leave a Reply

Your email address will not be published.