Solved!
This is the code:
First I used 'Number' as the first parameter of substringParameters, this didn't work. Also DebtorFields.Number.Name was not correct. The only correct form is to use it a the "IEntityField" representation.
// Below based on http://www.llblgen.com/TinyForum/Messages.aspx?ThreadID=14331
// Create custom field
object[] substringParameters = { DebtorFields.Number, 3, 20 };
DbFunctionCall substringCall = new DbFunctionCall("SUBSTRING", substringParameters);
EntityField sortField = new EntityField("NumberOnly", substringCall);
SortClause numberSortClause = new SortClause(sortField, null, SortOperator.Descending);
numberSortClause.EmitAliasForExpressionAggregateField = false;
SortExpression sort = new SortExpression(numberSortClause);
//object maxNumber = debtors.GetScalar(DebtorFieldIndex.DebtorId, null, AggregateFunction.Max, filter);
debtors.GetMulti(null, 1, sort);