I'm not sure if the exes "compiled" by AutoIt3 conform to the standard.
Excellent! Works in detecting CUI/GUI compiled AutoIt...now to apply my meager C knowledge in working out how they do it

EDIT: Here's a AutoIt conversion, not the whole file.exe program, just up to the part I was interested in. Seems to work OK as long as the AutoIt program hasn't been UPX'd of course.
Now to turn it into a function.
#Region ;**** Directives created by AutoIt3Wrapper_GUI ****
#AutoIt3Wrapper_UseUpx=n
#AutoIt3Wrapper_UseX64=n
#AutoIt3Wrapper_Change2CUI=y
#EndRegion ;**** Directives created by AutoIt3Wrapper_GUI ****
; file.au3
$maxfilesize = 8192
Dim $buffer[$maxfilesize]
If $CmdLine[0] = 0 Then Exit
$hFile = FileOpen($CmdLine[1], 16)
If $hFile = -1 Then
MsgBox(0, 'Error!', 'Failed to open file.')
Exit
EndIf
For $i = 0 To 8191
$buffer[$i] = Dec(StringMid(FileRead($hFile, 1), 3))
If @error Then ExitLoop
Next
FileClose($hFile)
; Test for executable
If ($buffer[0]=77 And $buffer[1]=90) Or ($buffer[0]=90 And $buffer[1]=77) Then ; MZ or ZM
ConsoleWrite('DOS/Win executable')
Else
ConsoleWrite('Not DOS/Win executable')
EndIf
$offset=0
$format=0
; Test for NE/PE format
$neoffset=($buffer[61] + $buffer[60]) * 256
If ($buffer[$neoffset] = 78 And $buffer[$neoffset + 1] = 69) Then
$format = 3
$offset = $neoffset
EndIf
$peoffset = $buffer[60]
If ($peoffset > $maxfilesize) Then
$peoffset = 0
EndIf
If ($buffer[$peoffset] = 80 And $buffer[$peoffset + 1] = 69) Then
$format = 1
$offset = $peoffset
EndIf
If ($format = 0) Then
$i = 0
While (($format = 0) And ($i < $maxfilesize - 4))
If ($buffer[$i] = 80) And ($buffer[$i + 1] = 69) And ($buffer[$i + 2] = 0) And ($buffer[$i + 3] = 0) Then
$offset = $i
$format = 2
EndIf
$i += 1
WEnd
EndIf
; Depending on the type of Format try to gain further information about the file
If ($format = 1 Or $format = 2) Then
If ($buffer[$offset + 4] = 0 And $buffer[$offset + 5] = 0) Then ConsoleWrite(', defined as an ANY / unknown machine type executable')
If ($buffer[$offset + 4] = 76 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', i386 32bit executable')
If ($buffer[$offset + 4] = 100 And $buffer[$offset + 5] = 134) Then ConsoleWrite(', x86-64 64bit executable')
If ($buffer[$offset + 4] = 0 And $buffer[$offset + 5] = 2) Then ConsoleWrite(', Intel Itanium executable')
If ($buffer[$offset + 4] = 240 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Power PC (little endian) executable')
If ($buffer[$offset + 4] = 241 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Power PC (with floating point support) executable')
If ($buffer[$offset + 4] = 211 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Matsushita AM33')
If ($buffer[$offset + 4] = 192 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', ARM little endian')
If ($buffer[$offset + 4] = 188 And $buffer[$offset + 5] = 14) Then ConsoleWrite(', EFI byte code')
If ($buffer[$offset + 4] = 65 And $buffer[$offset + 5] = 144) Then ConsoleWrite(', Mitsubishi M32R little endian')
If ($buffer[$offset + 4] = 102 And $buffer[$offset + 5] = 2) Then ConsoleWrite(', MIPS16')
If ($buffer[$offset + 4] = 102 And $buffer[$offset + 5] = 3) Then ConsoleWrite(', MIPS with FPU')
If ($buffer[$offset + 4] = 102 And $buffer[$offset + 5] = 4) Then ConsoleWrite(', MIPS 16 with FPU')
If ($buffer[$offset + 4] = 102 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', MIPS little endian / R4000')
If ($buffer[$offset + 4] = 162 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Hitachi SH3')
If ($buffer[$offset + 4] = 163 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Hitachi SH3 DSP')
If ($buffer[$offset + 4] = 166 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Hitachi SH4')
If ($buffer[$offset + 4] = 168 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Hitachi SH5')
If ($buffer[$offset + 4] = 194 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Thumb')
If ($buffer[$offset + 4] = 105 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', MIPS little-endian WCE v2')
If ($buffer[$offset + 4] = 132 And $buffer[$offset + 5] = 1) Then ConsoleWrite(', Alpha')
If ($buffer[$offset + 4] = 104 And $buffer[$offset + 5] = 2) Then ConsoleWrite(', Motorola 68000')
If ($buffer[$offset + 4] = 144 And $buffer[$offset + 5] = 2) Then ConsoleWrite(', PA-RISC')
If ($buffer[$offset + 23] = 32 And $buffer[$offset + 22] > 0) Then ConsoleWrite(', dll')
If ($buffer[$offset + 23] = 10 And $buffer[$offset + 22] > 0) Then ConsoleWrite(', system file')
If ($buffer[$offset + 92] = 0 And $buffer[$offset + 93] = 0) Then ConsoleWrite(', unknown system')
If ($buffer[$offset + 92] = 1 And $buffer[$offset + 93] = 0) Then ConsoleWrite(', native')
If ($buffer[$offset + 92] = 2 And $buffer[$offset + 93] = 0) Then ConsoleWrite(', GUI')
If ($buffer[$offset + 92] = 3 And $buffer[$offset + 93] = 0) Then ConsoleWrite(', console/cmd')
If ($buffer[$offset + 92] = 7 And $buffer[$offset + 93] = 0) Then ConsoleWrite(', POSIX')
EndIf
Exit
EDIT: Got sidetracked, while it will tell how the program was compiled it still won't tell you how it was run...but at least I can make a single program that covers both compilation options.
Here's a function, returns True if compiled for GUI, False for anything else.
MsgBox(0, '', _GUITest(@ScriptFullPath))
Func _GUITest($file)
Local $maxfilesize = 8192, $hFile, $offset, $buffer, $i, $format, $neoffset, $peoffset
Dim $buffer[$maxfilesize]
$hFile = FileOpen($file, 16)
For $i = 0 To 8191
$buffer[$i] = Dec(StringMid(FileRead($hFile, 1), 3))
If @error Then ExitLoop
Next
FileClose($hFile)
$offset=0
$format=0
$peoffset = $buffer[60]
If ($peoffset > $maxfilesize) Then
$peoffset = 0
EndIf
If ($buffer[$peoffset] = 80 And $buffer[$peoffset + 1] = 69) Then
$format = 1
$offset = $peoffset
EndIf
If ($format = 0) Then
$i = 0
While (($format = 0) And ($i < $maxfilesize - 4))
If ($buffer[$i] = 80) And ($buffer[$i + 1] = 69) And ($buffer[$i + 2] = 0) And ($buffer[$i + 3] = 0) Then
$offset = $i
$format = 2
EndIf
$i += 1
WEnd
EndIf
; Depending on the type of Format try to gain further information about the file
If ($format = 1 Or $format = 2) Then
If ($buffer[$offset + 92] = 2 And $buffer[$offset + 93] = 0) Then
Return True
Else
Return False
EndIf
EndIf
EndFunc