FileToRead=textfile.txt
FileToWrite=out.txt
;this version only supports a text file with the maximum size of 64mb



SetBatchLines,-1

IfExist,%filetowrite%
  {
  MsgBox,4,Question,Do you want to overwrite the file %filetowrite%?,30000
  IfMsgBox,Yes 
    FileDelete,%filetowrite%
  else
    exitapp
  }
FileRead,CompleteFile,%FileToRead%
StringSplit,index,CompleteFile,`n,`r`n
found=
count:=index0
count2:=count
GoSub,CreateGui2
ProgressFlag:=false
loop,%count%
  {
  GuiControl,2:,bar,%A_Index%
  If ProgressFlag
    break
  position:=A_Index
  Word:=index%position%
  If word is space
  {
    FileAppend,%word%`n,%filetowrite%
    continue
  }
  IfInString,found,%Word%
    continue
  count2-=1
  loop, %count2%
    {
    position2:=position+a_index
    Word2:=index%position2%
    if Word=%word2%
      found=%found% %Word% ,
    } 
  Fileappend,%Word%`n,%filetowrite%
  }
if found=
  msgbox,nothing was found
else
  {
  StringTrimRight,found2,found,2
  msgbox,The following strings were repeated: %found2%
  }
ExitApp


CreateGui2:
  Gui, 2:Add,Text,,Now checking for duplicate entries. Press esc to skip.
  Gui, 2:Add, Progress,vbar w300 h20 -smooth Range0-%count%,
  Gui, 2:Show
  return
  
2GuiClose:
  exitapp
  
2GuiEscape:
  Gui, 2:destroy
  ProgressFlag:=true
  return