For a .xlsm or .dotm file type you need to do it a slightly different way.

1. Change the extension of the .xlsm file to .zip.

2. Open the .zip file (with WinZip or WinRar etc) and go to the xl folder.
3. Extract the vbaProject.bin file and open it in a Hex Editor (I use HxD, its
completely free and lightweight.)
4. Search for DPB and replace with DPx and save the file.
5. Replace the old vbaProject.bin file with this new on in the zipped file.
6. Change the file extension back to .xlsm.
7. Open workbook skip through the warning messages.
8. Open up Visual Basic inside Excel.
9. Go to Tools > VBAProject Properties > Protection Tab.
10. Put in a new password and save the .xlsm file.
11. Close and re open and your new password will work.

Open xls file with a hex editor. (ie. Hex Edit http://www.hexedit.com/)
Search for DPB
Replace DPB to DPx
Save file.
Open file in Excel.
Click "Yes" if you get any message box.
Set new password from VBA Project Properties.
Close and open again file, then type your new password to unprotect.
UPDATE: For Excel 2010 (Works for MS Office Pro Plus 2010 [14.0.6023.1000 64bit]),

Open the XLSX file with 7zip

If workbook is protected:

Browse the folder xl

If the workbook is protected, right click workbook.xml and select Edit
Find the portion <workbookProtection workbookPassword="XXXX" lockStructure="1"/>
(XXXX is your encrypted password)
Remove XXXX part. (ie. <workbookProtection workbookPassword="" lockStructure="1"/>)
Save the file.
When 7zip asks you to update the archive, say Yes.
Close 7zip and re-open your XLSX.
Click Protect Workbook on Review tab.
Optional: Save your file.
If worksheets are protected:

Browse to xl/worksheets/ folder.

Right click the Sheet1.xml, sheet2.xml, etc and select Edit.
Find the portion <sheetProtection password="XXXX" sheet="1" objects="1"
scenarios="1" />
Remove the encrypted password (ie. <sheetProtection password="" sheet="1"
objects="1" scenarios="1" />)
Save the file.
When 7zip asks you to update the archive, say Yes.
Close 7zip and re-open your XLSX.
Click Unprotect Sheet on Review tab.
Optional: Save your file.

'**Break Password from VBA**

Option Explicit
Private Const PAGE_EXECUTE_READWRITE = &H40

Private Declare Sub MoveMemory Lib "kernel32" Alias "RtlMoveMemory" _

(Destination As Long, Source As Long, ByVal Length As Long)

Private Declare Function VirtualProtect Lib "kernel32" (lpAddress As Long, _

ByVal dwSize As Long, ByVal flNewProtect As Long, lpflOldProtect As Long)
As Long

Private Declare Function GetModuleHandleA Lib "kernel32" (ByVal lpModuleName As

String) As Long

Private Declare Function GetProcAddress Lib "kernel32" (ByVal hModule As Long, _

ByVal lpProcName As String) As Long

Private Declare Function DialogBoxParam Lib "user32" Alias "DialogBoxParamA" (ByVal

hInstance As Long, _
ByVal pTemplateName As Long, ByVal hWndParent As Long, _
ByVal lpDialogFunc As Long, ByVal dwInitParam As Long) As Integer

Dim HookBytes(0 To 5) As Byte

Dim OriginBytes(0 To 5) As Byte
Dim pFunc As Long
Dim Flag As Boolean

Private Function GetPtr(ByVal Value As Long) As Long

GetPtr = Value
End Function

Public Sub RecoverBytes()

If Flag Then MoveMemory ByVal pFunc, ByVal VarPtr(OriginBytes(0)), 6
End Sub

Public Function Hook() As Boolean

Dim TmpBytes(0 To 5) As Byte
Dim p As Long
Dim OriginProtect As Long

Hook = False

pFunc = GetProcAddress(GetModuleHandleA("user32.dll"), "DialogBoxParamA")

If VirtualProtect(ByVal pFunc, 6, PAGE_EXECUTE_READWRITE, OriginProtect) <> 0

Then

MoveMemory ByVal VarPtr(TmpBytes(0)), ByVal pFunc, 6

If TmpBytes(0) <> &H68 Then

MoveMemory ByVal VarPtr(OriginBytes(0)), ByVal pFunc, 6

p = GetPtr(AddressOf MyDialogBoxParam)

HookBytes(0) = &H68
MoveMemory ByVal VarPtr(HookBytes(1)), ByVal VarPtr(p), 4
HookBytes(5) = &HC3

MoveMemory ByVal pFunc, ByVal VarPtr(HookBytes(0)), 6

Flag = True
 Hook = True
End If
End If
End Function

Private Function MyDialogBoxParam(ByVal hInstance As Long, _

ByVal pTemplateName As Long, ByVal hWndParent As Long, _
ByVal lpDialogFunc As Long, ByVal dwInitParam As Long) As Integer
If pTemplateName = 4070 Then
MyDialogBoxParam = 1
Else
RecoverBytes
MyDialogBoxParam = DialogBoxParam(hInstance, pTemplateName, _
hWndParent, lpDialogFunc, dwInitParam)
Hook
End If
End Function

Sub unprotected()
If Hook Then
MsgBox "VBA Project is unprotected!", vbInformation, "*****"
End If
End Sub